Merge pull request #11003 from danfengliu/modify_api_test_scan_image_2

Modify API test for scan image since harbor v2 API presented

.github/workflows/CI.yml

@@ -112,6 +112,7 @@ jobs:
           pwd
           go env
           echo "::set-env name=CNAB_PATH::$(go env GOPATH)/src/github.com/docker"
+          echo "::set-env name=GITHUB_TOKEN::${{ secrets.GITHUB_TOKEN }}"
           echo "::set-env name=GOPATH::$(go env GOPATH):$GITHUB_WORKSPACE"
           echo "::add-path::$(go env GOPATH)/bin"
           echo "::set-env name=TOKEN_PRIVATE_KEY_PATH::${GITHUB_WORKSPACE}/src/github.com/goharbor/harbor/tests/private_key.pem"

tests/apitests/python/library/artifact.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 
+import time
 import base
 import v2_swagger_client
 from v2_swagger_client.rest import ApiException
@@ -12,6 +13,8 @@ class Artifact(base.Base):
             params["with_signature"] = kwargs["with_signature"]
         if "with_tag" in kwargs:
             params["with_tag"] = kwargs["with_tag"]
+        if "with_scan_overview" in kwargs:
+            params["with_scan_overview"] = kwargs["with_scan_overview"]
         return client.get_artifact_with_http_info(project_name, repo_name, reference, **params )
 
     def add_label_to_reference(self, project_name, repo_name, reference, label_id, **kwargs):
@@ -44,3 +47,17 @@ class Artifact(base.Base):
         client = self._get_client(**kwargs)
         _, status_code, _ = client.delete_tag_with_http_info(project_name, repo_name, reference, tag_name)
         base._assert_status_code(expect_status_code, status_code)
+
+    def check_image_scan_result(self, project_name, repo_name, reference, expected_scan_status = "Success", **kwargs):
+        timeout_count = 30
+        scan_status=""
+        while True:
+            time.sleep(5)
+            timeout_count = timeout_count - 1
+            if (timeout_count == 0):
+                break
+            artifact = self.get_reference_info(project_name, repo_name, reference, **kwargs)
+            scan_status = artifact[0].scan_overview['application/vnd.scanner.adapter.vuln.report.harbor+json; version=1.0']["scan_status"]
+            if scan_status == expected_scan_status:
+                return
+        raise Exception("Scan image result is {}, not as expected {}.".format(scan_status, expected_scan_status))

tests/apitests/python/library/base.py

@@ -38,6 +38,7 @@ def _create_client(server, credential, debug, api_type="products"):
         "products":   swagger_client.ProductsApi(swagger_client.ApiClient(cfg)),
         "artifact":   v2_swagger_client.ArtifactApi(v2_swagger_client.ApiClient(cfg)),
         "repository": v2_swagger_client.RepositoryApi(v2_swagger_client.ApiClient(cfg)),
+        "scan": v2_swagger_client.ScanApi(v2_swagger_client.ApiClient(cfg)),
     }.get(api_type,'Error: Wrong API type')
 
 def _assert_status_code(expect_code, return_code):

tests/apitests/python/library/repository.py

@@ -107,26 +107,6 @@ class Repository(base.Base):
         if tag.scan_overview != None:
             raise Exception("Image should be <Not Scanned> state!")
 
-    def check_image_scan_result(self, repo_name, tag, expected_scan_status = "Success", **kwargs):
-        timeout_count = 30
-        while True:
-            time.sleep(5)
-            timeout_count = timeout_count - 1
-            if (timeout_count == 0):
-                break
-            _tag = self.get_tag(repo_name, tag, **kwargs)
-            if _tag.name == tag and _tag.scan_overview != None:
-                for report in _tag.scan_overview.values():
-                    if report.get('scan_status') == expected_scan_status:
-                        return
-        raise Exception("Scan image result is not as expected {}.".format(expected_scan_status))
-
-    def scan_image(self, repo_name, tag, expect_status_code = 202, **kwargs):
-        client = self._get_client(**kwargs)
-        data, status_code, _ = client.repositories_repo_name_tags_tag_scan_post_with_http_info(repo_name, tag)
-        base._assert_status_code(expect_status_code, status_code)
-        return data
-
     def repository_should_exist(self, project_id, repo_name, **kwargs):
         repositories = self.list_repositories(project_id, **kwargs)
         if is_repo_exist_in_project(repositories, repo_name) == False:

tests/apitests/python/library/scan.py

@@ -0,0 +1,13 @@
+# -*- coding: utf-8 -*-
+
+import time
+import base
+import v2_swagger_client
+from v2_swagger_client.rest import ApiException
+
+class Scan(base.Base):
+    def scan_artifact(self, project_name, repo_name, reference, expect_status_code = 202, **kwargs):
+        client = self._get_client(**kwargs)
+        data, status_code, _ = client.scan_artifact_with_http_info(project_name, repo_name, reference)
+        base._assert_status_code(expect_status_code, status_code)
+        return data

tests/apitests/python/test_scan_image_artifact.py

@@ -8,22 +8,25 @@ from library.project import Project
 from library.user import User
 from library.repository import Repository
 from library.repository import push_image_to_project
-
+from library.artifact import Artifact
+from library.scan import Scan
 class TestProjects(unittest.TestCase):
     @classmethod
     def setUp(self):
         self.project= Project()
         self.user= User()
-        self.repo= Repository()
+        self.artifact = Artifact(api_type='artifact')
+        self.repo = Repository(api_type='repository')
+        self.scan = Scan(api_type='scan')
 
     @classmethod
     def tearDown(self):
         print "Case completed"
 
-    @unittest.skipIf(TEARDOWN == True, "Test data won't be erased.")
+    @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.")
     def test_ClearData(self):
         #1. Delete repository(RA) by user(UA);
-        self.repo.delete_repoitory(TestProjects.repo_name, **TestProjects.USER_SCAN_IMAGE_CLIENT)
+        self.repo.delete_repoitory(TestProjects.project_scan_image_name, TestProjects.repo_name.split('/')[1], **TestProjects.USER_SCAN_IMAGE_CLIENT)
 
         #2. Delete project(PA);
         self.project.delete_project(TestProjects.project_scan_image_id, **TestProjects.USER_SCAN_IMAGE_CLIENT)
@@ -31,10 +34,10 @@ class TestProjects(unittest.TestCase):
         #3. Delete user(UA);
         self.user.delete_user(TestProjects.user_scan_image_id, **ADMIN_CLIENT)
 
-    def testScanImage(self):
+    def testScanImageArtifact(self):
         """
         Test case:
-            Scan A Image
+            Scan An Image Artifact
         Test step and expected result:
             1. Create a new user(UA);
             2. Create a new private project(PA) by user(UA);
@@ -53,10 +56,10 @@ class TestProjects(unittest.TestCase):
         #1. Create user-001
         TestProjects.user_scan_image_id, user_scan_image_name = self.user.create_user(user_password = user_001_password, **ADMIN_CLIENT)
 
-        TestProjects.USER_SCAN_IMAGE_CLIENT=dict(endpoint = url, username = user_scan_image_name, password = user_001_password)
+        TestProjects.USER_SCAN_IMAGE_CLIENT=dict(endpoint = url, username = user_scan_image_name, password = user_001_password, with_scan_overview = True)
 
         #2. Create a new private project(PA) by user(UA);
-        TestProjects.project_scan_image_id, project_scan_image_name = self.project.create_project(metadata = {"public": "false"}, **ADMIN_CLIENT)
+        TestProjects.project_scan_image_id, TestProjects.project_scan_image_name = self.project.create_project(metadata = {"public": "false"}, **ADMIN_CLIENT)
 
         #3. Add user(UA) as a member of project(PA) with project-admin role;
         self.project.add_project_members(TestProjects.project_scan_image_id, TestProjects.user_scan_image_id, **ADMIN_CLIENT)
@@ -71,11 +74,13 @@ class TestProjects(unittest.TestCase):
         image = "docker"
         src_tag = "1.13"
         #5. Create a new repository(RA) and tag(TA) in project(PA) by user(UA);
-        TestProjects.repo_name, tag = push_image_to_project(project_scan_image_name, harbor_server, user_scan_image_name, user_001_password, image, src_tag)
+        TestProjects.repo_name, tag = push_image_to_project(TestProjects.project_scan_image_name, harbor_server, user_scan_image_name, user_001_password, image, src_tag)
 
         #6. Send scan image command and get tag(TA) information to check scan result, it should be finished;
-        self.repo.scan_image(TestProjects.repo_name, tag, **TestProjects.USER_SCAN_IMAGE_CLIENT)
+        self.scan.scan_artifact(TestProjects.project_scan_image_name, TestProjects.repo_name.split('/')[1], tag, **TestProjects.USER_SCAN_IMAGE_CLIENT)
-        self.repo.check_image_scan_result(TestProjects.repo_name, tag, **TestProjects.USER_SCAN_IMAGE_CLIENT)
+
+        #6. Send scan image command and get tag(TA) information to check scan result, it should be finished;
+        self.artifact.check_image_scan_result(TestProjects.project_scan_image_name, image, tag, **TestProjects.USER_SCAN_IMAGE_CLIENT)
 
 if __name__ == '__main__':
     unittest.main()

tests/apitests/python/test_system_level_scan_all.py

@@ -9,6 +9,7 @@ from library.project import Project
 from library.user import User
 from library.repository import Repository
 from library.repository import push_image_to_project
+from library.artifact import Artifact
 
 class TestProjects(unittest.TestCase):
     @classmethod
@@ -16,7 +17,8 @@ class TestProjects(unittest.TestCase):
         self.system = System()
         self.project= Project()
         self.user= User()
-        self.repo= Repository()
+        self.artifact = Artifact(api_type='artifact')
+        self.repo = Repository(api_type='repository')
 
     @classmethod
     def tearDown(self):
@@ -25,8 +27,8 @@ class TestProjects(unittest.TestCase):
     @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.")
     def test_ClearData(self):
         #1. Delete Alice's repository and Luca's repository;
-        self.repo.delete_repoitory(TestProjects.repo_Alice_name, **ADMIN_CLIENT)
+        self.repo.delete_repoitory(TestProjects.project_Alice_name, TestProjects.repo_Alice_name.split('/')[1], **ADMIN_CLIENT)
-        self.repo.delete_repoitory(TestProjects.repo_Luca_name, **ADMIN_CLIENT)
+        self.repo.delete_repoitory(TestProjects.project_Luca_name, TestProjects.repo_Luca_name.split('/')[1], **ADMIN_CLIENT)
 
         #2. Delete Alice's project and Luca's project;
         self.project.delete_project(TestProjects.project_Alice_id, **ADMIN_CLIENT)
@@ -36,10 +38,10 @@ class TestProjects(unittest.TestCase):
         self.user.delete_user(TestProjects.user_Alice_id, **ADMIN_CLIENT)
         self.user.delete_user(TestProjects.user_Luca_id, **ADMIN_CLIENT)
 
-    def testScanImage(self):
+    def testSystemLevelScanALL(self):
         """
         Test case:
-            Scan All Image
+            System level Scan All
         Test step and expected result:
             1. Create user Alice and Luca;
             2. Create 2 new private projects project_Alice and project_Luca;
@@ -58,36 +60,38 @@ class TestProjects(unittest.TestCase):
         TestProjects.user_Alice_id, user_Alice_name = self.user.create_user(user_password = user_common_password, **ADMIN_CLIENT)
         TestProjects.user_Luca_id, user_Luca_name = self.user.create_user(user_password = user_common_password, **ADMIN_CLIENT)
 
-        USER_ALICE_CLIENT=dict(endpoint = url, username = user_Alice_name, password = user_common_password)
+        USER_ALICE_CLIENT=dict(endpoint = url, username = user_Alice_name, password = user_common_password, with_scan_overview = True)
-        USER_LUCA_CLIENT=dict(endpoint = url, username = user_Luca_name, password = user_common_password)
+        USER_LUCA_CLIENT=dict(endpoint = url, username = user_Luca_name, password = user_common_password, with_scan_overview = True)
 
         #2. Create 2 new private projects project_Alice and project_Luca;
-        TestProjects.project_Alice_id, project_Alice_name = self.project.create_project(metadata = {"public": "false"}, **USER_ALICE_CLIENT)
+        TestProjects.project_Alice_id, TestProjects.project_Alice_name = self.project.create_project(metadata = {"public": "false"}, **USER_ALICE_CLIENT)
-        TestProjects.project_Luca_id, project_Luca_name = self.project.create_project(metadata = {"public": "false"}, **USER_LUCA_CLIENT)
+        TestProjects.project_Luca_id, TestProjects.project_Luca_name = self.project.create_project(metadata = {"public": "false"}, **USER_LUCA_CLIENT)
 
         #3. Push a image to project_Alice and push another image to project_Luca;
 
         #Note: Please make sure that this Image has never been pulled before by any other cases,
         #          so it is a not-scanned image rigth after repository creation.
         #image = "tomcat"
-        image = "mariadb"
+        image_a = "mariadb"
         src_tag = "latest"
         #3.1 Push a image to project_Alice;
-        TestProjects.repo_Alice_name, tag_Alice = push_image_to_project(project_Alice_name, harbor_server, user_Alice_name, user_common_password, image, src_tag)
+        TestProjects.repo_Alice_name, tag_Alice = push_image_to_project(TestProjects.project_Alice_name, harbor_server, user_Alice_name, user_common_password, image_a, src_tag)
 
         #Note: Please make sure that this Image has never been pulled before by any other cases,
         #          so it is a not-scanned image rigth after repository creation.
-        image = "httpd"
+        image_b = "httpd"
         src_tag = "latest"
         #3.2 push another image to project_Luca;
-        TestProjects.repo_Luca_name, tag_Luca = push_image_to_project(project_Luca_name, harbor_server, user_Luca_name, user_common_password, image, src_tag)
+        TestProjects.repo_Luca_name, tag_Luca = push_image_to_project(TestProjects.project_Luca_name, harbor_server, user_Luca_name, user_common_password, image_b, src_tag)
 
         #4. Trigger scan all event;
         self.system.scan_now(**ADMIN_CLIENT)
 
         #5. Check if image in project_Alice and another image in project_Luca were both scanned.
-        self.repo.check_image_scan_result(TestProjects.repo_Alice_name, tag_Alice, **USER_ALICE_CLIENT)
+        #self.repo.check_image_scan_result(TestProjects.repo_Alice_name, tag_Alice, **USER_ALICE_CLIENT)
-        self.repo.check_image_scan_result(TestProjects.repo_Luca_name, tag_Luca, **USER_LUCA_CLIENT)
+        #self.repo.check_image_scan_result(TestProjects.repo_Luca_name, tag_Luca, **USER_LUCA_CLIENT)
+        self.artifact.check_image_scan_result(TestProjects.project_Alice_name, image_a, tag_Alice, **USER_ALICE_CLIENT)
+        self.artifact.check_image_scan_result(TestProjects.project_Luca_name, image_b, tag_Luca, **USER_LUCA_CLIENT)
 
 if __name__ == '__main__':
     unittest.main()

tests/ci/api_common_install.sh

@@ -26,5 +26,9 @@ sudo curl -o $DIR/../../tests/apitests/python/mariadb-4.3.1.tgz https://storage.
 sudo apt-get update && sudo apt-get install -y --no-install-recommends python-dev openjdk-7-jdk libssl-dev && sudo apt-get autoremove -y && sudo rm -rf /var/lib/apt/lists/*
 sudo wget https://bootstrap.pypa.io/get-pip.py && sudo python ./get-pip.py && sudo pip install --ignore-installed urllib3 chardet requests && sudo pip install robotframework==3.0.4 robotframework-httplibrary requests dbbot robotframework-pabot --upgrade
 sudo make swagger_client
-sudo make install GOBUILDIMAGE=golang:1.13.8 COMPILETAG=compile_golangimage CLARITYIMAGE=goharbor/harbor-clarity-ui-builder:1.6.0 NOTARYFLAG=true CLAIRFLAG=true TRIVYFLAG=true CHARTFLAG=true
+if [ $GITHUB_TOKEN ];
+then
+    sed "s/# github_token: xxx/github_token: $GITHUB_TOKEN/" -i make/harbor.yml
+fi
+sudo make install GOBUILDIMAGE=golang:1.13.4 COMPILETAG=compile_golangimage CLARITYIMAGE=goharbor/harbor-clarity-ui-builder:1.6.0 NOTARYFLAG=true CLAIRFLAG=true TRIVYFLAG=true CHARTFLAG=true
 sleep 10

tests/robot-cases/Group0-BAT/API_DB.robot

@@ -29,9 +29,10 @@ Test Case - Add Replication Rule
     Harbor API Test  ./tests/apitests/python/test_add_replication_rule.py
 Test Case - Edit Project Creation
     Harbor API Test  ./tests/apitests/python/test_edit_project_creation.py
-# TODO uncomment this after image scan work with basic auth - #10277
+Test Case - Scan Image
-#Test Case - Scan Image
+    Harbor API Test  ./tests/apitests/python/test_scan_image_artifact.py
-#    Harbor API Test  ./tests/apitests/python/test_scan_image.py
+Test Case - Scan All Images
+    Harbor API Test  ./tests/apitests/python/test_system_level_scan_all.py
 Test Case - Manage Project Member
     Harbor API Test  ./tests/apitests/python/test_manage_project_member.py
 Test Case - Project Level Policy Content Trust
@@ -41,9 +42,6 @@ Test Case - Project Level Policy Content Trust
 # User View Logs still in failure state - danfeng@3/11 2020.
 # Test Case - User View Logs
 #    Harbor API Test  ./tests/apitests/python/test_user_view_logs.py
-# TODO uncomment this after making scan all work with OCI registry
-# Test Case - Scan All Images
-#     Harbor API Test  ./tests/apitests/python/test_scan_all_images.py
 Test Case - List Helm Charts
     Harbor API Test  ./tests/apitests/python/test_list_helm_charts.py
 Test Case - Assign Sys Admin