Merge pull request #11003 from danfengliu/modify_api_test_scan_image_2

Modify API test for scan image since harbor v2 API presented
This commit is contained in:
Wenkai Yin(尹文开) 2020-03-18 11:00:55 +08:00 committed by GitHub
commit b0e87b46e4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 76 additions and 53 deletions

View File

@ -112,6 +112,7 @@ jobs:
pwd pwd
go env go env
echo "::set-env name=CNAB_PATH::$(go env GOPATH)/src/github.com/docker" echo "::set-env name=CNAB_PATH::$(go env GOPATH)/src/github.com/docker"
echo "::set-env name=GITHUB_TOKEN::${{ secrets.GITHUB_TOKEN }}"
echo "::set-env name=GOPATH::$(go env GOPATH):$GITHUB_WORKSPACE" echo "::set-env name=GOPATH::$(go env GOPATH):$GITHUB_WORKSPACE"
echo "::add-path::$(go env GOPATH)/bin" echo "::add-path::$(go env GOPATH)/bin"
echo "::set-env name=TOKEN_PRIVATE_KEY_PATH::${GITHUB_WORKSPACE}/src/github.com/goharbor/harbor/tests/private_key.pem" echo "::set-env name=TOKEN_PRIVATE_KEY_PATH::${GITHUB_WORKSPACE}/src/github.com/goharbor/harbor/tests/private_key.pem"

View File

@ -1,5 +1,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import time
import base import base
import v2_swagger_client import v2_swagger_client
from v2_swagger_client.rest import ApiException from v2_swagger_client.rest import ApiException
@ -12,6 +13,8 @@ class Artifact(base.Base):
params["with_signature"] = kwargs["with_signature"] params["with_signature"] = kwargs["with_signature"]
if "with_tag" in kwargs: if "with_tag" in kwargs:
params["with_tag"] = kwargs["with_tag"] params["with_tag"] = kwargs["with_tag"]
if "with_scan_overview" in kwargs:
params["with_scan_overview"] = kwargs["with_scan_overview"]
return client.get_artifact_with_http_info(project_name, repo_name, reference, **params ) return client.get_artifact_with_http_info(project_name, repo_name, reference, **params )
def add_label_to_reference(self, project_name, repo_name, reference, label_id, **kwargs): def add_label_to_reference(self, project_name, repo_name, reference, label_id, **kwargs):
@ -44,3 +47,17 @@ class Artifact(base.Base):
client = self._get_client(**kwargs) client = self._get_client(**kwargs)
_, status_code, _ = client.delete_tag_with_http_info(project_name, repo_name, reference, tag_name) _, status_code, _ = client.delete_tag_with_http_info(project_name, repo_name, reference, tag_name)
base._assert_status_code(expect_status_code, status_code) base._assert_status_code(expect_status_code, status_code)
def check_image_scan_result(self, project_name, repo_name, reference, expected_scan_status = "Success", **kwargs):
timeout_count = 30
scan_status=""
while True:
time.sleep(5)
timeout_count = timeout_count - 1
if (timeout_count == 0):
break
artifact = self.get_reference_info(project_name, repo_name, reference, **kwargs)
scan_status = artifact[0].scan_overview['application/vnd.scanner.adapter.vuln.report.harbor+json; version=1.0']["scan_status"]
if scan_status == expected_scan_status:
return
raise Exception("Scan image result is {}, not as expected {}.".format(scan_status, expected_scan_status))

View File

@ -38,6 +38,7 @@ def _create_client(server, credential, debug, api_type="products"):
"products": swagger_client.ProductsApi(swagger_client.ApiClient(cfg)), "products": swagger_client.ProductsApi(swagger_client.ApiClient(cfg)),
"artifact": v2_swagger_client.ArtifactApi(v2_swagger_client.ApiClient(cfg)), "artifact": v2_swagger_client.ArtifactApi(v2_swagger_client.ApiClient(cfg)),
"repository": v2_swagger_client.RepositoryApi(v2_swagger_client.ApiClient(cfg)), "repository": v2_swagger_client.RepositoryApi(v2_swagger_client.ApiClient(cfg)),
"scan": v2_swagger_client.ScanApi(v2_swagger_client.ApiClient(cfg)),
}.get(api_type,'Error: Wrong API type') }.get(api_type,'Error: Wrong API type')
def _assert_status_code(expect_code, return_code): def _assert_status_code(expect_code, return_code):

View File

@ -107,26 +107,6 @@ class Repository(base.Base):
if tag.scan_overview != None: if tag.scan_overview != None:
raise Exception("Image should be <Not Scanned> state!") raise Exception("Image should be <Not Scanned> state!")
def check_image_scan_result(self, repo_name, tag, expected_scan_status = "Success", **kwargs):
timeout_count = 30
while True:
time.sleep(5)
timeout_count = timeout_count - 1
if (timeout_count == 0):
break
_tag = self.get_tag(repo_name, tag, **kwargs)
if _tag.name == tag and _tag.scan_overview != None:
for report in _tag.scan_overview.values():
if report.get('scan_status') == expected_scan_status:
return
raise Exception("Scan image result is not as expected {}.".format(expected_scan_status))
def scan_image(self, repo_name, tag, expect_status_code = 202, **kwargs):
client = self._get_client(**kwargs)
data, status_code, _ = client.repositories_repo_name_tags_tag_scan_post_with_http_info(repo_name, tag)
base._assert_status_code(expect_status_code, status_code)
return data
def repository_should_exist(self, project_id, repo_name, **kwargs): def repository_should_exist(self, project_id, repo_name, **kwargs):
repositories = self.list_repositories(project_id, **kwargs) repositories = self.list_repositories(project_id, **kwargs)
if is_repo_exist_in_project(repositories, repo_name) == False: if is_repo_exist_in_project(repositories, repo_name) == False:

View File

@ -0,0 +1,13 @@
# -*- coding: utf-8 -*-
import time
import base
import v2_swagger_client
from v2_swagger_client.rest import ApiException
class Scan(base.Base):
def scan_artifact(self, project_name, repo_name, reference, expect_status_code = 202, **kwargs):
client = self._get_client(**kwargs)
data, status_code, _ = client.scan_artifact_with_http_info(project_name, repo_name, reference)
base._assert_status_code(expect_status_code, status_code)
return data

View File

@ -8,22 +8,25 @@ from library.project import Project
from library.user import User from library.user import User
from library.repository import Repository from library.repository import Repository
from library.repository import push_image_to_project from library.repository import push_image_to_project
from library.artifact import Artifact
from library.scan import Scan
class TestProjects(unittest.TestCase): class TestProjects(unittest.TestCase):
@classmethod @classmethod
def setUp(self): def setUp(self):
self.project= Project() self.project= Project()
self.user= User() self.user= User()
self.repo= Repository() self.artifact = Artifact(api_type='artifact')
self.repo = Repository(api_type='repository')
self.scan = Scan(api_type='scan')
@classmethod @classmethod
def tearDown(self): def tearDown(self):
print "Case completed" print "Case completed"
@unittest.skipIf(TEARDOWN == True, "Test data won't be erased.") @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.")
def test_ClearData(self): def test_ClearData(self):
#1. Delete repository(RA) by user(UA); #1. Delete repository(RA) by user(UA);
self.repo.delete_repoitory(TestProjects.repo_name, **TestProjects.USER_SCAN_IMAGE_CLIENT) self.repo.delete_repoitory(TestProjects.project_scan_image_name, TestProjects.repo_name.split('/')[1], **TestProjects.USER_SCAN_IMAGE_CLIENT)
#2. Delete project(PA); #2. Delete project(PA);
self.project.delete_project(TestProjects.project_scan_image_id, **TestProjects.USER_SCAN_IMAGE_CLIENT) self.project.delete_project(TestProjects.project_scan_image_id, **TestProjects.USER_SCAN_IMAGE_CLIENT)
@ -31,10 +34,10 @@ class TestProjects(unittest.TestCase):
#3. Delete user(UA); #3. Delete user(UA);
self.user.delete_user(TestProjects.user_scan_image_id, **ADMIN_CLIENT) self.user.delete_user(TestProjects.user_scan_image_id, **ADMIN_CLIENT)
def testScanImage(self): def testScanImageArtifact(self):
""" """
Test case: Test case:
Scan A Image Scan An Image Artifact
Test step and expected result: Test step and expected result:
1. Create a new user(UA); 1. Create a new user(UA);
2. Create a new private project(PA) by user(UA); 2. Create a new private project(PA) by user(UA);
@ -53,10 +56,10 @@ class TestProjects(unittest.TestCase):
#1. Create user-001 #1. Create user-001
TestProjects.user_scan_image_id, user_scan_image_name = self.user.create_user(user_password = user_001_password, **ADMIN_CLIENT) TestProjects.user_scan_image_id, user_scan_image_name = self.user.create_user(user_password = user_001_password, **ADMIN_CLIENT)
TestProjects.USER_SCAN_IMAGE_CLIENT=dict(endpoint = url, username = user_scan_image_name, password = user_001_password) TestProjects.USER_SCAN_IMAGE_CLIENT=dict(endpoint = url, username = user_scan_image_name, password = user_001_password, with_scan_overview = True)
#2. Create a new private project(PA) by user(UA); #2. Create a new private project(PA) by user(UA);
TestProjects.project_scan_image_id, project_scan_image_name = self.project.create_project(metadata = {"public": "false"}, **ADMIN_CLIENT) TestProjects.project_scan_image_id, TestProjects.project_scan_image_name = self.project.create_project(metadata = {"public": "false"}, **ADMIN_CLIENT)
#3. Add user(UA) as a member of project(PA) with project-admin role; #3. Add user(UA) as a member of project(PA) with project-admin role;
self.project.add_project_members(TestProjects.project_scan_image_id, TestProjects.user_scan_image_id, **ADMIN_CLIENT) self.project.add_project_members(TestProjects.project_scan_image_id, TestProjects.user_scan_image_id, **ADMIN_CLIENT)
@ -71,11 +74,13 @@ class TestProjects(unittest.TestCase):
image = "docker" image = "docker"
src_tag = "1.13" src_tag = "1.13"
#5. Create a new repository(RA) and tag(TA) in project(PA) by user(UA); #5. Create a new repository(RA) and tag(TA) in project(PA) by user(UA);
TestProjects.repo_name, tag = push_image_to_project(project_scan_image_name, harbor_server, user_scan_image_name, user_001_password, image, src_tag) TestProjects.repo_name, tag = push_image_to_project(TestProjects.project_scan_image_name, harbor_server, user_scan_image_name, user_001_password, image, src_tag)
#6. Send scan image command and get tag(TA) information to check scan result, it should be finished; #6. Send scan image command and get tag(TA) information to check scan result, it should be finished;
self.repo.scan_image(TestProjects.repo_name, tag, **TestProjects.USER_SCAN_IMAGE_CLIENT) self.scan.scan_artifact(TestProjects.project_scan_image_name, TestProjects.repo_name.split('/')[1], tag, **TestProjects.USER_SCAN_IMAGE_CLIENT)
self.repo.check_image_scan_result(TestProjects.repo_name, tag, **TestProjects.USER_SCAN_IMAGE_CLIENT)
#6. Send scan image command and get tag(TA) information to check scan result, it should be finished;
self.artifact.check_image_scan_result(TestProjects.project_scan_image_name, image, tag, **TestProjects.USER_SCAN_IMAGE_CLIENT)
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() unittest.main()

View File

@ -9,6 +9,7 @@ from library.project import Project
from library.user import User from library.user import User
from library.repository import Repository from library.repository import Repository
from library.repository import push_image_to_project from library.repository import push_image_to_project
from library.artifact import Artifact
class TestProjects(unittest.TestCase): class TestProjects(unittest.TestCase):
@classmethod @classmethod
@ -16,7 +17,8 @@ class TestProjects(unittest.TestCase):
self.system = System() self.system = System()
self.project= Project() self.project= Project()
self.user= User() self.user= User()
self.repo= Repository() self.artifact = Artifact(api_type='artifact')
self.repo = Repository(api_type='repository')
@classmethod @classmethod
def tearDown(self): def tearDown(self):
@ -25,8 +27,8 @@ class TestProjects(unittest.TestCase):
@unittest.skipIf(TEARDOWN == False, "Test data won't be erased.") @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.")
def test_ClearData(self): def test_ClearData(self):
#1. Delete Alice's repository and Luca's repository; #1. Delete Alice's repository and Luca's repository;
self.repo.delete_repoitory(TestProjects.repo_Alice_name, **ADMIN_CLIENT) self.repo.delete_repoitory(TestProjects.project_Alice_name, TestProjects.repo_Alice_name.split('/')[1], **ADMIN_CLIENT)
self.repo.delete_repoitory(TestProjects.repo_Luca_name, **ADMIN_CLIENT) self.repo.delete_repoitory(TestProjects.project_Luca_name, TestProjects.repo_Luca_name.split('/')[1], **ADMIN_CLIENT)
#2. Delete Alice's project and Luca's project; #2. Delete Alice's project and Luca's project;
self.project.delete_project(TestProjects.project_Alice_id, **ADMIN_CLIENT) self.project.delete_project(TestProjects.project_Alice_id, **ADMIN_CLIENT)
@ -36,10 +38,10 @@ class TestProjects(unittest.TestCase):
self.user.delete_user(TestProjects.user_Alice_id, **ADMIN_CLIENT) self.user.delete_user(TestProjects.user_Alice_id, **ADMIN_CLIENT)
self.user.delete_user(TestProjects.user_Luca_id, **ADMIN_CLIENT) self.user.delete_user(TestProjects.user_Luca_id, **ADMIN_CLIENT)
def testScanImage(self): def testSystemLevelScanALL(self):
""" """
Test case: Test case:
Scan All Image System level Scan All
Test step and expected result: Test step and expected result:
1. Create user Alice and Luca; 1. Create user Alice and Luca;
2. Create 2 new private projects project_Alice and project_Luca; 2. Create 2 new private projects project_Alice and project_Luca;
@ -58,36 +60,38 @@ class TestProjects(unittest.TestCase):
TestProjects.user_Alice_id, user_Alice_name = self.user.create_user(user_password = user_common_password, **ADMIN_CLIENT) TestProjects.user_Alice_id, user_Alice_name = self.user.create_user(user_password = user_common_password, **ADMIN_CLIENT)
TestProjects.user_Luca_id, user_Luca_name = self.user.create_user(user_password = user_common_password, **ADMIN_CLIENT) TestProjects.user_Luca_id, user_Luca_name = self.user.create_user(user_password = user_common_password, **ADMIN_CLIENT)
USER_ALICE_CLIENT=dict(endpoint = url, username = user_Alice_name, password = user_common_password) USER_ALICE_CLIENT=dict(endpoint = url, username = user_Alice_name, password = user_common_password, with_scan_overview = True)
USER_LUCA_CLIENT=dict(endpoint = url, username = user_Luca_name, password = user_common_password) USER_LUCA_CLIENT=dict(endpoint = url, username = user_Luca_name, password = user_common_password, with_scan_overview = True)
#2. Create 2 new private projects project_Alice and project_Luca; #2. Create 2 new private projects project_Alice and project_Luca;
TestProjects.project_Alice_id, project_Alice_name = self.project.create_project(metadata = {"public": "false"}, **USER_ALICE_CLIENT) TestProjects.project_Alice_id, TestProjects.project_Alice_name = self.project.create_project(metadata = {"public": "false"}, **USER_ALICE_CLIENT)
TestProjects.project_Luca_id, project_Luca_name = self.project.create_project(metadata = {"public": "false"}, **USER_LUCA_CLIENT) TestProjects.project_Luca_id, TestProjects.project_Luca_name = self.project.create_project(metadata = {"public": "false"}, **USER_LUCA_CLIENT)
#3. Push a image to project_Alice and push another image to project_Luca; #3. Push a image to project_Alice and push another image to project_Luca;
#Note: Please make sure that this Image has never been pulled before by any other cases, #Note: Please make sure that this Image has never been pulled before by any other cases,
# so it is a not-scanned image rigth after repository creation. # so it is a not-scanned image rigth after repository creation.
#image = "tomcat" #image = "tomcat"
image = "mariadb" image_a = "mariadb"
src_tag = "latest" src_tag = "latest"
#3.1 Push a image to project_Alice; #3.1 Push a image to project_Alice;
TestProjects.repo_Alice_name, tag_Alice = push_image_to_project(project_Alice_name, harbor_server, user_Alice_name, user_common_password, image, src_tag) TestProjects.repo_Alice_name, tag_Alice = push_image_to_project(TestProjects.project_Alice_name, harbor_server, user_Alice_name, user_common_password, image_a, src_tag)
#Note: Please make sure that this Image has never been pulled before by any other cases, #Note: Please make sure that this Image has never been pulled before by any other cases,
# so it is a not-scanned image rigth after repository creation. # so it is a not-scanned image rigth after repository creation.
image = "httpd" image_b = "httpd"
src_tag = "latest" src_tag = "latest"
#3.2 push another image to project_Luca; #3.2 push another image to project_Luca;
TestProjects.repo_Luca_name, tag_Luca = push_image_to_project(project_Luca_name, harbor_server, user_Luca_name, user_common_password, image, src_tag) TestProjects.repo_Luca_name, tag_Luca = push_image_to_project(TestProjects.project_Luca_name, harbor_server, user_Luca_name, user_common_password, image_b, src_tag)
#4. Trigger scan all event; #4. Trigger scan all event;
self.system.scan_now(**ADMIN_CLIENT) self.system.scan_now(**ADMIN_CLIENT)
#5. Check if image in project_Alice and another image in project_Luca were both scanned. #5. Check if image in project_Alice and another image in project_Luca were both scanned.
self.repo.check_image_scan_result(TestProjects.repo_Alice_name, tag_Alice, **USER_ALICE_CLIENT) #self.repo.check_image_scan_result(TestProjects.repo_Alice_name, tag_Alice, **USER_ALICE_CLIENT)
self.repo.check_image_scan_result(TestProjects.repo_Luca_name, tag_Luca, **USER_LUCA_CLIENT) #self.repo.check_image_scan_result(TestProjects.repo_Luca_name, tag_Luca, **USER_LUCA_CLIENT)
self.artifact.check_image_scan_result(TestProjects.project_Alice_name, image_a, tag_Alice, **USER_ALICE_CLIENT)
self.artifact.check_image_scan_result(TestProjects.project_Luca_name, image_b, tag_Luca, **USER_LUCA_CLIENT)
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() unittest.main()

View File

@ -26,5 +26,9 @@ sudo curl -o $DIR/../../tests/apitests/python/mariadb-4.3.1.tgz https://storage.
sudo apt-get update && sudo apt-get install -y --no-install-recommends python-dev openjdk-7-jdk libssl-dev && sudo apt-get autoremove -y && sudo rm -rf /var/lib/apt/lists/* sudo apt-get update && sudo apt-get install -y --no-install-recommends python-dev openjdk-7-jdk libssl-dev && sudo apt-get autoremove -y && sudo rm -rf /var/lib/apt/lists/*
sudo wget https://bootstrap.pypa.io/get-pip.py && sudo python ./get-pip.py && sudo pip install --ignore-installed urllib3 chardet requests && sudo pip install robotframework==3.0.4 robotframework-httplibrary requests dbbot robotframework-pabot --upgrade sudo wget https://bootstrap.pypa.io/get-pip.py && sudo python ./get-pip.py && sudo pip install --ignore-installed urllib3 chardet requests && sudo pip install robotframework==3.0.4 robotframework-httplibrary requests dbbot robotframework-pabot --upgrade
sudo make swagger_client sudo make swagger_client
sudo make install GOBUILDIMAGE=golang:1.13.8 COMPILETAG=compile_golangimage CLARITYIMAGE=goharbor/harbor-clarity-ui-builder:1.6.0 NOTARYFLAG=true CLAIRFLAG=true TRIVYFLAG=true CHARTFLAG=true if [ $GITHUB_TOKEN ];
then
sed "s/# github_token: xxx/github_token: $GITHUB_TOKEN/" -i make/harbor.yml
fi
sudo make install GOBUILDIMAGE=golang:1.13.4 COMPILETAG=compile_golangimage CLARITYIMAGE=goharbor/harbor-clarity-ui-builder:1.6.0 NOTARYFLAG=true CLAIRFLAG=true TRIVYFLAG=true CHARTFLAG=true
sleep 10 sleep 10

View File

@ -29,9 +29,10 @@ Test Case - Add Replication Rule
Harbor API Test ./tests/apitests/python/test_add_replication_rule.py Harbor API Test ./tests/apitests/python/test_add_replication_rule.py
Test Case - Edit Project Creation Test Case - Edit Project Creation
Harbor API Test ./tests/apitests/python/test_edit_project_creation.py Harbor API Test ./tests/apitests/python/test_edit_project_creation.py
# TODO uncomment this after image scan work with basic auth - #10277 Test Case - Scan Image
#Test Case - Scan Image Harbor API Test ./tests/apitests/python/test_scan_image_artifact.py
# Harbor API Test ./tests/apitests/python/test_scan_image.py Test Case - Scan All Images
Harbor API Test ./tests/apitests/python/test_system_level_scan_all.py
Test Case - Manage Project Member Test Case - Manage Project Member
Harbor API Test ./tests/apitests/python/test_manage_project_member.py Harbor API Test ./tests/apitests/python/test_manage_project_member.py
Test Case - Project Level Policy Content Trust Test Case - Project Level Policy Content Trust
@ -41,9 +42,6 @@ Test Case - Project Level Policy Content Trust
# User View Logs still in failure state - danfeng@3/11 2020. # User View Logs still in failure state - danfeng@3/11 2020.
# Test Case - User View Logs # Test Case - User View Logs
# Harbor API Test ./tests/apitests/python/test_user_view_logs.py # Harbor API Test ./tests/apitests/python/test_user_view_logs.py
# TODO uncomment this after making scan all work with OCI registry
# Test Case - Scan All Images
# Harbor API Test ./tests/apitests/python/test_scan_all_images.py
Test Case - List Helm Charts Test Case - List Helm Charts
Harbor API Test ./tests/apitests/python/test_list_helm_charts.py Harbor API Test ./tests/apitests/python/test_list_helm_charts.py
Test Case - Assign Sys Admin Test Case - Assign Sys Admin