diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 9dbdb74bf..7e10ddec6 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -112,6 +112,7 @@ jobs: pwd go env echo "::set-env name=CNAB_PATH::$(go env GOPATH)/src/github.com/docker" + echo "::set-env name=GITHUB_TOKEN::${{ secrets.GITHUB_TOKEN }}" echo "::set-env name=GOPATH::$(go env GOPATH):$GITHUB_WORKSPACE" echo "::add-path::$(go env GOPATH)/bin" echo "::set-env name=TOKEN_PRIVATE_KEY_PATH::${GITHUB_WORKSPACE}/src/github.com/goharbor/harbor/tests/private_key.pem" diff --git a/tests/apitests/python/library/artifact.py b/tests/apitests/python/library/artifact.py index 5c3f05850..8a6ed1a68 100644 --- a/tests/apitests/python/library/artifact.py +++ b/tests/apitests/python/library/artifact.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- +import time import base import v2_swagger_client from v2_swagger_client.rest import ApiException @@ -12,6 +13,8 @@ class Artifact(base.Base): params["with_signature"] = kwargs["with_signature"] if "with_tag" in kwargs: params["with_tag"] = kwargs["with_tag"] + if "with_scan_overview" in kwargs: + params["with_scan_overview"] = kwargs["with_scan_overview"] return client.get_artifact_with_http_info(project_name, repo_name, reference, **params ) def add_label_to_reference(self, project_name, repo_name, reference, label_id, **kwargs): @@ -44,3 +47,17 @@ class Artifact(base.Base): client = self._get_client(**kwargs) _, status_code, _ = client.delete_tag_with_http_info(project_name, repo_name, reference, tag_name) base._assert_status_code(expect_status_code, status_code) + + def check_image_scan_result(self, project_name, repo_name, reference, expected_scan_status = "Success", **kwargs): + timeout_count = 30 + scan_status="" + while True: + time.sleep(5) + timeout_count = timeout_count - 1 + if (timeout_count == 0): + break + artifact = self.get_reference_info(project_name, repo_name, reference, **kwargs) + scan_status = artifact[0].scan_overview['application/vnd.scanner.adapter.vuln.report.harbor+json; version=1.0']["scan_status"] + if scan_status == expected_scan_status: + return + raise Exception("Scan image result is {}, not as expected {}.".format(scan_status, expected_scan_status)) diff --git a/tests/apitests/python/library/base.py b/tests/apitests/python/library/base.py index 62dcc2b56..561e842df 100644 --- a/tests/apitests/python/library/base.py +++ b/tests/apitests/python/library/base.py @@ -38,6 +38,7 @@ def _create_client(server, credential, debug, api_type="products"): "products": swagger_client.ProductsApi(swagger_client.ApiClient(cfg)), "artifact": v2_swagger_client.ArtifactApi(v2_swagger_client.ApiClient(cfg)), "repository": v2_swagger_client.RepositoryApi(v2_swagger_client.ApiClient(cfg)), + "scan": v2_swagger_client.ScanApi(v2_swagger_client.ApiClient(cfg)), }.get(api_type,'Error: Wrong API type') def _assert_status_code(expect_code, return_code): diff --git a/tests/apitests/python/library/repository.py b/tests/apitests/python/library/repository.py index 47c6f96c6..9f4925736 100644 --- a/tests/apitests/python/library/repository.py +++ b/tests/apitests/python/library/repository.py @@ -107,26 +107,6 @@ class Repository(base.Base): if tag.scan_overview != None: raise Exception("Image should be state!") - def check_image_scan_result(self, repo_name, tag, expected_scan_status = "Success", **kwargs): - timeout_count = 30 - while True: - time.sleep(5) - timeout_count = timeout_count - 1 - if (timeout_count == 0): - break - _tag = self.get_tag(repo_name, tag, **kwargs) - if _tag.name == tag and _tag.scan_overview != None: - for report in _tag.scan_overview.values(): - if report.get('scan_status') == expected_scan_status: - return - raise Exception("Scan image result is not as expected {}.".format(expected_scan_status)) - - def scan_image(self, repo_name, tag, expect_status_code = 202, **kwargs): - client = self._get_client(**kwargs) - data, status_code, _ = client.repositories_repo_name_tags_tag_scan_post_with_http_info(repo_name, tag) - base._assert_status_code(expect_status_code, status_code) - return data - def repository_should_exist(self, project_id, repo_name, **kwargs): repositories = self.list_repositories(project_id, **kwargs) if is_repo_exist_in_project(repositories, repo_name) == False: diff --git a/tests/apitests/python/library/scan.py b/tests/apitests/python/library/scan.py new file mode 100644 index 000000000..4a65f8735 --- /dev/null +++ b/tests/apitests/python/library/scan.py @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- + +import time +import base +import v2_swagger_client +from v2_swagger_client.rest import ApiException + +class Scan(base.Base): + def scan_artifact(self, project_name, repo_name, reference, expect_status_code = 202, **kwargs): + client = self._get_client(**kwargs) + data, status_code, _ = client.scan_artifact_with_http_info(project_name, repo_name, reference) + base._assert_status_code(expect_status_code, status_code) + return data diff --git a/tests/apitests/python/test_scan_image.py b/tests/apitests/python/test_scan_image_artifact.py similarity index 68% rename from tests/apitests/python/test_scan_image.py rename to tests/apitests/python/test_scan_image_artifact.py index fe2a09b8c..d7afe4c35 100644 --- a/tests/apitests/python/test_scan_image.py +++ b/tests/apitests/python/test_scan_image_artifact.py @@ -8,22 +8,25 @@ from library.project import Project from library.user import User from library.repository import Repository from library.repository import push_image_to_project - +from library.artifact import Artifact +from library.scan import Scan class TestProjects(unittest.TestCase): @classmethod def setUp(self): self.project= Project() self.user= User() - self.repo= Repository() + self.artifact = Artifact(api_type='artifact') + self.repo = Repository(api_type='repository') + self.scan = Scan(api_type='scan') @classmethod def tearDown(self): print "Case completed" - @unittest.skipIf(TEARDOWN == True, "Test data won't be erased.") + @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.") def test_ClearData(self): #1. Delete repository(RA) by user(UA); - self.repo.delete_repoitory(TestProjects.repo_name, **TestProjects.USER_SCAN_IMAGE_CLIENT) + self.repo.delete_repoitory(TestProjects.project_scan_image_name, TestProjects.repo_name.split('/')[1], **TestProjects.USER_SCAN_IMAGE_CLIENT) #2. Delete project(PA); self.project.delete_project(TestProjects.project_scan_image_id, **TestProjects.USER_SCAN_IMAGE_CLIENT) @@ -31,10 +34,10 @@ class TestProjects(unittest.TestCase): #3. Delete user(UA); self.user.delete_user(TestProjects.user_scan_image_id, **ADMIN_CLIENT) - def testScanImage(self): + def testScanImageArtifact(self): """ Test case: - Scan A Image + Scan An Image Artifact Test step and expected result: 1. Create a new user(UA); 2. Create a new private project(PA) by user(UA); @@ -53,10 +56,10 @@ class TestProjects(unittest.TestCase): #1. Create user-001 TestProjects.user_scan_image_id, user_scan_image_name = self.user.create_user(user_password = user_001_password, **ADMIN_CLIENT) - TestProjects.USER_SCAN_IMAGE_CLIENT=dict(endpoint = url, username = user_scan_image_name, password = user_001_password) + TestProjects.USER_SCAN_IMAGE_CLIENT=dict(endpoint = url, username = user_scan_image_name, password = user_001_password, with_scan_overview = True) #2. Create a new private project(PA) by user(UA); - TestProjects.project_scan_image_id, project_scan_image_name = self.project.create_project(metadata = {"public": "false"}, **ADMIN_CLIENT) + TestProjects.project_scan_image_id, TestProjects.project_scan_image_name = self.project.create_project(metadata = {"public": "false"}, **ADMIN_CLIENT) #3. Add user(UA) as a member of project(PA) with project-admin role; self.project.add_project_members(TestProjects.project_scan_image_id, TestProjects.user_scan_image_id, **ADMIN_CLIENT) @@ -71,11 +74,13 @@ class TestProjects(unittest.TestCase): image = "docker" src_tag = "1.13" #5. Create a new repository(RA) and tag(TA) in project(PA) by user(UA); - TestProjects.repo_name, tag = push_image_to_project(project_scan_image_name, harbor_server, user_scan_image_name, user_001_password, image, src_tag) + TestProjects.repo_name, tag = push_image_to_project(TestProjects.project_scan_image_name, harbor_server, user_scan_image_name, user_001_password, image, src_tag) #6. Send scan image command and get tag(TA) information to check scan result, it should be finished; - self.repo.scan_image(TestProjects.repo_name, tag, **TestProjects.USER_SCAN_IMAGE_CLIENT) - self.repo.check_image_scan_result(TestProjects.repo_name, tag, **TestProjects.USER_SCAN_IMAGE_CLIENT) + self.scan.scan_artifact(TestProjects.project_scan_image_name, TestProjects.repo_name.split('/')[1], tag, **TestProjects.USER_SCAN_IMAGE_CLIENT) + + #6. Send scan image command and get tag(TA) information to check scan result, it should be finished; + self.artifact.check_image_scan_result(TestProjects.project_scan_image_name, image, tag, **TestProjects.USER_SCAN_IMAGE_CLIENT) if __name__ == '__main__': unittest.main() \ No newline at end of file diff --git a/tests/apitests/python/test_scan_all_images.py b/tests/apitests/python/test_system_level_scan_all.py similarity index 67% rename from tests/apitests/python/test_scan_all_images.py rename to tests/apitests/python/test_system_level_scan_all.py index b8cd9e29f..f897e31fc 100644 --- a/tests/apitests/python/test_scan_all_images.py +++ b/tests/apitests/python/test_system_level_scan_all.py @@ -9,6 +9,7 @@ from library.project import Project from library.user import User from library.repository import Repository from library.repository import push_image_to_project +from library.artifact import Artifact class TestProjects(unittest.TestCase): @classmethod @@ -16,7 +17,8 @@ class TestProjects(unittest.TestCase): self.system = System() self.project= Project() self.user= User() - self.repo= Repository() + self.artifact = Artifact(api_type='artifact') + self.repo = Repository(api_type='repository') @classmethod def tearDown(self): @@ -25,8 +27,8 @@ class TestProjects(unittest.TestCase): @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.") def test_ClearData(self): #1. Delete Alice's repository and Luca's repository; - self.repo.delete_repoitory(TestProjects.repo_Alice_name, **ADMIN_CLIENT) - self.repo.delete_repoitory(TestProjects.repo_Luca_name, **ADMIN_CLIENT) + self.repo.delete_repoitory(TestProjects.project_Alice_name, TestProjects.repo_Alice_name.split('/')[1], **ADMIN_CLIENT) + self.repo.delete_repoitory(TestProjects.project_Luca_name, TestProjects.repo_Luca_name.split('/')[1], **ADMIN_CLIENT) #2. Delete Alice's project and Luca's project; self.project.delete_project(TestProjects.project_Alice_id, **ADMIN_CLIENT) @@ -36,10 +38,10 @@ class TestProjects(unittest.TestCase): self.user.delete_user(TestProjects.user_Alice_id, **ADMIN_CLIENT) self.user.delete_user(TestProjects.user_Luca_id, **ADMIN_CLIENT) - def testScanImage(self): + def testSystemLevelScanALL(self): """ Test case: - Scan All Image + System level Scan All Test step and expected result: 1. Create user Alice and Luca; 2. Create 2 new private projects project_Alice and project_Luca; @@ -58,36 +60,38 @@ class TestProjects(unittest.TestCase): TestProjects.user_Alice_id, user_Alice_name = self.user.create_user(user_password = user_common_password, **ADMIN_CLIENT) TestProjects.user_Luca_id, user_Luca_name = self.user.create_user(user_password = user_common_password, **ADMIN_CLIENT) - USER_ALICE_CLIENT=dict(endpoint = url, username = user_Alice_name, password = user_common_password) - USER_LUCA_CLIENT=dict(endpoint = url, username = user_Luca_name, password = user_common_password) + USER_ALICE_CLIENT=dict(endpoint = url, username = user_Alice_name, password = user_common_password, with_scan_overview = True) + USER_LUCA_CLIENT=dict(endpoint = url, username = user_Luca_name, password = user_common_password, with_scan_overview = True) #2. Create 2 new private projects project_Alice and project_Luca; - TestProjects.project_Alice_id, project_Alice_name = self.project.create_project(metadata = {"public": "false"}, **USER_ALICE_CLIENT) - TestProjects.project_Luca_id, project_Luca_name = self.project.create_project(metadata = {"public": "false"}, **USER_LUCA_CLIENT) + TestProjects.project_Alice_id, TestProjects.project_Alice_name = self.project.create_project(metadata = {"public": "false"}, **USER_ALICE_CLIENT) + TestProjects.project_Luca_id, TestProjects.project_Luca_name = self.project.create_project(metadata = {"public": "false"}, **USER_LUCA_CLIENT) #3. Push a image to project_Alice and push another image to project_Luca; #Note: Please make sure that this Image has never been pulled before by any other cases, # so it is a not-scanned image rigth after repository creation. #image = "tomcat" - image = "mariadb" + image_a = "mariadb" src_tag = "latest" #3.1 Push a image to project_Alice; - TestProjects.repo_Alice_name, tag_Alice = push_image_to_project(project_Alice_name, harbor_server, user_Alice_name, user_common_password, image, src_tag) + TestProjects.repo_Alice_name, tag_Alice = push_image_to_project(TestProjects.project_Alice_name, harbor_server, user_Alice_name, user_common_password, image_a, src_tag) #Note: Please make sure that this Image has never been pulled before by any other cases, # so it is a not-scanned image rigth after repository creation. - image = "httpd" + image_b = "httpd" src_tag = "latest" #3.2 push another image to project_Luca; - TestProjects.repo_Luca_name, tag_Luca = push_image_to_project(project_Luca_name, harbor_server, user_Luca_name, user_common_password, image, src_tag) + TestProjects.repo_Luca_name, tag_Luca = push_image_to_project(TestProjects.project_Luca_name, harbor_server, user_Luca_name, user_common_password, image_b, src_tag) #4. Trigger scan all event; self.system.scan_now(**ADMIN_CLIENT) #5. Check if image in project_Alice and another image in project_Luca were both scanned. - self.repo.check_image_scan_result(TestProjects.repo_Alice_name, tag_Alice, **USER_ALICE_CLIENT) - self.repo.check_image_scan_result(TestProjects.repo_Luca_name, tag_Luca, **USER_LUCA_CLIENT) + #self.repo.check_image_scan_result(TestProjects.repo_Alice_name, tag_Alice, **USER_ALICE_CLIENT) + #self.repo.check_image_scan_result(TestProjects.repo_Luca_name, tag_Luca, **USER_LUCA_CLIENT) + self.artifact.check_image_scan_result(TestProjects.project_Alice_name, image_a, tag_Alice, **USER_ALICE_CLIENT) + self.artifact.check_image_scan_result(TestProjects.project_Luca_name, image_b, tag_Luca, **USER_LUCA_CLIENT) if __name__ == '__main__': unittest.main() \ No newline at end of file diff --git a/tests/ci/api_common_install.sh b/tests/ci/api_common_install.sh index 574c798a9..42d66f4f9 100755 --- a/tests/ci/api_common_install.sh +++ b/tests/ci/api_common_install.sh @@ -26,5 +26,9 @@ sudo curl -o $DIR/../../tests/apitests/python/mariadb-4.3.1.tgz https://storage. sudo apt-get update && sudo apt-get install -y --no-install-recommends python-dev openjdk-7-jdk libssl-dev && sudo apt-get autoremove -y && sudo rm -rf /var/lib/apt/lists/* sudo wget https://bootstrap.pypa.io/get-pip.py && sudo python ./get-pip.py && sudo pip install --ignore-installed urllib3 chardet requests && sudo pip install robotframework==3.0.4 robotframework-httplibrary requests dbbot robotframework-pabot --upgrade sudo make swagger_client -sudo make install GOBUILDIMAGE=golang:1.13.8 COMPILETAG=compile_golangimage CLARITYIMAGE=goharbor/harbor-clarity-ui-builder:1.6.0 NOTARYFLAG=true CLAIRFLAG=true TRIVYFLAG=true CHARTFLAG=true +if [ $GITHUB_TOKEN ]; +then + sed "s/# github_token: xxx/github_token: $GITHUB_TOKEN/" -i make/harbor.yml +fi +sudo make install GOBUILDIMAGE=golang:1.13.4 COMPILETAG=compile_golangimage CLARITYIMAGE=goharbor/harbor-clarity-ui-builder:1.6.0 NOTARYFLAG=true CLAIRFLAG=true TRIVYFLAG=true CHARTFLAG=true sleep 10 diff --git a/tests/robot-cases/Group0-BAT/API_DB.robot b/tests/robot-cases/Group0-BAT/API_DB.robot index 5895797fb..8fbe670e4 100644 --- a/tests/robot-cases/Group0-BAT/API_DB.robot +++ b/tests/robot-cases/Group0-BAT/API_DB.robot @@ -29,9 +29,10 @@ Test Case - Add Replication Rule Harbor API Test ./tests/apitests/python/test_add_replication_rule.py Test Case - Edit Project Creation Harbor API Test ./tests/apitests/python/test_edit_project_creation.py -# TODO uncomment this after image scan work with basic auth - #10277 -#Test Case - Scan Image -# Harbor API Test ./tests/apitests/python/test_scan_image.py +Test Case - Scan Image + Harbor API Test ./tests/apitests/python/test_scan_image_artifact.py +Test Case - Scan All Images + Harbor API Test ./tests/apitests/python/test_system_level_scan_all.py Test Case - Manage Project Member Harbor API Test ./tests/apitests/python/test_manage_project_member.py Test Case - Project Level Policy Content Trust @@ -41,9 +42,6 @@ Test Case - Project Level Policy Content Trust # User View Logs still in failure state - danfeng@3/11 2020. # Test Case - User View Logs # Harbor API Test ./tests/apitests/python/test_user_view_logs.py -# TODO uncomment this after making scan all work with OCI registry -# Test Case - Scan All Images -# Harbor API Test ./tests/apitests/python/test_scan_all_images.py Test Case - List Helm Charts Harbor API Test ./tests/apitests/python/test_list_helm_charts.py Test Case - Assign Sys Admin