Merge pull request #12265 from danfengliu/add-gc-untag-pytest

Add API E2E pytest of GC with untag flag enabled

tests/apitests/python/library/helm.py

@@ -35,7 +35,6 @@ def helm_chart_push_to_harbor(chart_file, archive, harbor_server, project, repo_
     helm_save(archive, harbor_server, project, repo_name)
     return helm_push(harbor_server, project, repo_name, version)
 
-# helm repo add --ca-file /ca/server.crt --username=${user} --password=${pwd} ${helm_repo_name} ${harbor_url}/chartrepo/${project_name}
 def helm2_add_repo(helm_repo_name, harbor_url, project, username, password):
     command = ["helm2", "repo", "add", "--username=" + username, "--password=" + password, helm_repo_name, harbor_url + "/chartrepo/" + project]
     print "Command: ", command

tests/apitests/python/library/project.py

@@ -183,7 +183,7 @@ class Project(base.Base):
         access_list = []
         resource_by_project_id = "/project/"+str(project_id)+"/repository"
         resource_helm_by_project_id = "/project/"+str(project_id)+"/helm-chart"
-        resource_helm__create_by_project_id = "/project/"+str(project_id)+"/helm-chart-version"
+        resource_helm_create_by_project_id = "/project/"+str(project_id)+"/helm-chart-version"
         action_pull = "pull"
         action_push = "push"
         action_read = "read"
@@ -198,7 +198,7 @@ class Project(base.Base):
             robotAccountAccess = swagger_client.RobotAccountAccess(resource = resource_helm_by_project_id, action = action_read)
             access_list.append(robotAccountAccess)
         if has_chart_create_right is True:
-            robotAccountAccess = swagger_client.RobotAccountAccess(resource = resource_helm__create_by_project_id, action = action_create)
+            robotAccountAccess = swagger_client.RobotAccountAccess(resource = resource_helm_create_by_project_id, action = action_create)
             access_list.append(robotAccountAccess)
 
         robotAccountCreate = swagger_client.RobotAccountCreate(robot_name, robot_desc, expires_at, access_list)

tests/apitests/python/test_garbage_collection.py

@@ -11,6 +11,8 @@ from library.repository import Repository
 from library.repository import push_image_to_project
 from testutils import harbor_server
 from library.base import _assert_status_code
+from library.repository import push_special_image_to_project
+from library.artifact import Artifact
 
 class TestProjects(unittest.TestCase):
     @classmethod
@@ -19,12 +21,16 @@ class TestProjects(unittest.TestCase):
         self.project = Project()
         self.user = User()
         self.repo = Repository()
+        self.artifact = Artifact()
+        self.repo_name = "test_repo"
+        self.repo_name_untag = "test_untag"
+        self.tag = "v1.0"
 
     @classmethod
     def tearDown(self):
         print "Case completed"
 
-    @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.")
+    @unittest.skipIf(TEARDOWN == True, "Test data won't be erased.")
     def test_ClearData(self):
         #2. Delete project(PA);
         self.project.delete_project(TestProjects.project_gc_id, **TestProjects.USER_GC_CLIENT)
@@ -38,13 +44,17 @@ class TestProjects(unittest.TestCase):
             Garbage Collection
         Test step and expected result:
             1. Create a new user(UA);
-            2. Create a new project(PA) by user(UA);
+            2. Create project(PA) and project(PB) by user(UA);
-            3. Push a new image(IA) in project(PA) by admin;
+            3. Push a image in project(PA) and then delete repository by admin;
-            4. Delete repository(RA) by user(UA);
+            4. Get repository by user(UA), it should get nothing;
-            5. Get repository by user(UA), it should get nothing;
+            5. Tigger garbage collection operation;
-            6. Tigger garbage collection operation;
+            6. Check garbage collection job was finished;
-            7. Check garbage collection job was finished;
+            7. Get garbage collection log, check there is a number of files was deleted;
-            8. Get garbage collection log, check there is number of files was deleted.
+            8. Push a image in project(PB) by admin and delete the only tag;
+            9. Tigger garbage collection operation;
+            10. Check garbage collection job was finished;
+            11. Repository with untag image should be still there;
+            12. But no any artifact in repository anymore.
         Tear down:
             1. Delete project(PA);
             2. Delete user(UA).
@@ -59,27 +69,48 @@ class TestProjects(unittest.TestCase):
 
         TestProjects.USER_GC_CLIENT=dict(endpoint = url, username = user_gc_name, password = user_gc_password)
 
-        #2. Create a new project(PA) by user(UA);
+        #2. Create project(PA) and project(PB) by user(UA);
         TestProjects.project_gc_id, TestProjects.project_gc_name = self.project.create_project(metadata = {"public": "false"}, **TestProjects.USER_GC_CLIENT)
+        TestProjects.project_gc_untag_id, TestProjects.project_gc_untag_name = self.project.create_project(metadata = {"public": "false"}, **TestProjects.USER_GC_CLIENT)
 
-        #3. Push a new image(IA) in project(PA) by admin;
+        #3. Push a image in project(PA) and then delete repository by admin;
-        repo_name, _ = push_image_to_project(TestProjects.project_gc_name, harbor_server, admin_name, admin_password, "tomcat", "latest")
+        push_special_image_to_project(TestProjects.project_gc_name, harbor_server, admin_name, admin_password, self.repo_name, ["latest", "v1.2.3"])
+        self.repo.delete_repoitory(TestProjects.project_gc_name, self.repo_name, **TestProjects.USER_GC_CLIENT)
 
-        #4. Delete repository(RA) by user(UA);
+        #4. Get repository by user(UA), it should get nothing;
-        self.repo.delete_repoitory(TestProjects.project_gc_name, repo_name.split('/')[1], **TestProjects.USER_GC_CLIENT)
-
-        #5. Get repository by user(UA), it should get nothing;
         repo_data = self.repo.list_repositories(TestProjects.project_gc_name, **TestProjects.USER_GC_CLIENT)
         _assert_status_code(len(repo_data), 0)
 
-        #6. Tigger garbage collection operation;
+        #5. Tigger garbage collection operation;
         gc_id = self.system.gc_now(**ADMIN_CLIENT)
 
-        #7. Check garbage collection job was finished;
+        #6. Check garbage collection job was finished;
         self.system.validate_gc_job_status(gc_id, "finished", **ADMIN_CLIENT)
 
-        #8. Get garbage collection log, check there is number of files was deleted.
+        #7. Get garbage collection log, check there is a number of files was deleted;
         self.system.validate_deletion_success(gc_id, **ADMIN_CLIENT)
 
+        #8. Push a image in project(PB) by admin and delete the only tag;
+        push_special_image_to_project(TestProjects.project_gc_untag_name, harbor_server, admin_name, admin_password, self.repo_name_untag, [self.tag])
+        self.artifact.delete_tag(TestProjects.project_gc_untag_name, self.repo_name_untag, self.tag, self.tag, **ADMIN_CLIENT)
+
+        #9. Tigger garbage collection operation;
+        gc_id = self.system.gc_now(**ADMIN_CLIENT)
+
+        #10. Check garbage collection job was finished;
+        self.system.validate_gc_job_status(gc_id, "finished", **ADMIN_CLIENT)
+
+        #11. Repository with untag image should be still there;
+        repo_data_untag = self.repo.list_repositories(TestProjects.project_gc_untag_name, **TestProjects.USER_GC_CLIENT)
+        print "repo_data_untag:", repo_data_untag
+        _assert_status_code(len(repo_data_untag), 1)
+        self.assertEqual(TestProjects.project_gc_untag_name + "/" + self.repo_name_untag , repo_data_untag[0].name)
+
+        #12. But no any artifact in repository anymore.
+        artifacts = self.artifact.list_artifacts(TestProjects.project_gc_untag_name, self.repo_name_untag, **TestProjects.USER_GC_CLIENT)
+        self.assertEqual(artifacts,[])
+
+
+
 if __name__ == '__main__':
     unittest.main()

tests/apitests/python/test_list_helm_charts.py

@@ -11,14 +11,9 @@ from library.chart import Chart
 class TestProjects(unittest.TestCase):
     @classmethod
     def setUp(self):
-        chart = Chart()
+        self.chart= Chart()
-        self.chart= chart
+        self.project= Project()
-
+        self.user= User()
-        project = Project()
-        self.project= project
-
-        user = User()
-        self.user= user
 
     @classmethod
     def tearDown(self):

tests/apitests/python/test_push_chart_by_helm2_helm3_with_robot_Account.py

@@ -5,26 +5,26 @@ import unittest
 
 import library.repository
 import library.helm
-from testutils import ADMIN_CLIENT
+from testutils import ADMIN_CLIENT, CHART_API_CLIENT
 from testutils import harbor_server
 
 from testutils import TEARDOWN
 from library.project import Project
 from library.user import User
-from library.repository import Repository
+from library.chart import Chart
-from library.artifact import Artifact
 
 class TestProjects(unittest.TestCase):
     @classmethod
     def setUpClass(self):
         self.project= Project()
         self.user= User()
-        self.artifact = Artifact()
+        self.chart=  Chart()
-        self.repo= Repository()
         self.url = ADMIN_CLIENT["endpoint"]
+        self.chart_api_url = CHART_API_CLIENT['endpoint']
         self.user_push_chart_password = "Aa123456"
         self.chart_file = "https://storage.googleapis.com/harbor-builds/helm-chart-test-files/harbor-0.2.0.tgz"
         self.archive = "harbor/"
+        self.CHART_NAME=self.archive.replace("/", "")
         self.verion = "0.2.0"
         self.chart_repo_name = "chart_local"
         self.repo_name = "harbor_api_test"
@@ -54,15 +54,15 @@ class TestProjects(unittest.TestCase):
 
         print "#1. Create user(UA);"
         TestProjects.user_id, user_name = self.user.create_user(user_password = self.user_push_chart_password, **ADMIN_CLIENT)
-        TestProjects.USER_RA_CLIENT=dict(endpoint = self.url, username = user_name, password = self.user_push_chart_password)
+        TestProjects.USER_CLIENT=dict(endpoint = self.url, username = user_name, password = self.user_push_chart_password)
-
+        TestProjects.API_CHART_CLIENT=dict(endpoint = self.chart_api_url, username = user_name, password = self.user_push_chart_password)
         print "#2. Create private project(PA) with user(UA);"
-        TestProjects.project_id, TestProjects.project_name = self.project.create_project(metadata = {"public": "false"}, **TestProjects.USER_RA_CLIENT)
+        TestProjects.project_id, TestProjects.project_name = self.project.create_project(metadata = {"public": "false"}, **TestProjects.USER_CLIENT)
 
 
         print "#3. Create a new robot account(RA) with full priviliges in project(PA) with user(UA);"
         robot_id, robot_account = self.project.add_project_robot_account(TestProjects.project_id, TestProjects.project_name,
-                                                                         2441000531 ,**TestProjects.USER_RA_CLIENT)
+                                                                         2441000531 ,**TestProjects.USER_CLIENT)
         print robot_account.name
         print robot_account.token
 
@@ -71,7 +71,7 @@ class TestProjects(unittest.TestCase):
         library.helm.helm2_push(self.chart_repo_name, self.chart_file, TestProjects.project_name, robot_account.name, robot_account.token)
 
         print "#5. Get chart repositry from project(PA) successfully;"
-        # Depend on issue #12252
+        self.chart.chart_should_exist(TestProjects.project_name, self.CHART_NAME, **TestProjects.API_CHART_CLIENT)
 
         print "#6. Push chart to project(PA) by Helm3 CLI with robot account(RA);"
         chart_cli_ret = library.helm.helm_chart_push_to_harbor(self.chart_file, self.archive,  harbor_server, TestProjects.project_name, self.repo_name, self.verion, robot_account.name, robot_account.token)

tests/apitests/python/test_user_view_logs.py

@@ -90,6 +90,7 @@ class TestProjects(unittest.TestCase):
                                              format(user_user_view_logs_name, project_user_view_logs_name, "artifact", operation, log_count))
         #4.1 Delete repository(RA) by user(UA);
         self.repo.delete_repoitory(project_user_view_logs_name, repo_name.split('/')[1], **TestProjects.USER_USER_VIEW_LOGS_CLIENT)
+        time.sleep(6)
 
         #4.2 In project(PA), there should be 1 'delete' log record;
         operation = "delete"

tests/resources/Harbor-Pages/Project-Artifact.robot

@@ -32,16 +32,15 @@ Should Not Contain Tag
 
 Add A New Tag
     [Arguments]  ${tag}
-    Retry Element Click   ${add_tag_button}
+    Retry Double Keywords When Error  Retry Element Click   ${add_tag_button}  Retry Wait Element  ${tag_name_xpath}
     Retry Text Input   ${tag_name_xpath}   ${tag}
-    Retry Element Click   ${add_ok_button}
+    Retry Double Keywords When Error  Retry Element Click   ${add_ok_button}  Should Contain Tag  ${tag}
 
 Delete A Tag
     [Arguments]  ${tag}
     Retry Element Click   xpath=//clr-dg-row[contains(.,'${tag}')]//clr-checkbox-wrapper//label[contains(@class,'clr-control-label')]
-    Retry Element Click    ${delete_tag_button}
+    Retry Double Keywords When Error  Retry Element Click    ${delete_tag_button}  Retry Wait Until Page Contains Element  ${dialog_delete_button}
-    Retry Wait Until Page Contains Element  ${dialog_delete_button}
+    Retry Double Keywords When Error  Retry Element Click  ${dialog_delete_button}  Should Not Contain Tag  ${tag}
-    Retry Element Click  ${dialog_delete_button}
 
 Should Contain Artifact
     Retry Wait Until Page Contains Element   xpath=//artifact-list-tab//clr-dg-row//a[contains(.,'sha256')]

tests/resources/Harbor-Pages/Project.robot

@@ -196,6 +196,18 @@ Do Log Advanced Search
     ${rc} =  Get Element Count  //audit-log//clr-dg-row
     Should Be Equal As Integers  ${rc}  0
 
+Retry Click Repo Name
+    [Arguments]  ${repo_name_element}
+    :For  ${n}  IN RANGE  1  10
+    \    ${out}  Run Keyword And Ignore Error  Retry Double Keywords When Error  Retry Element Click  ${repo_name_element}   Retry Wait Element  ${tag_table_column_vulnerabilities}
+    \    Exit For Loop If  '${out[0]}'=='PASS'
+    Should Be Equal As Strings  '${out[0]}'  'PASS'
+
+    :For  ${n}  IN RANGE  1  10
+    \    ${out}  Run Keyword And Ignore Error  Retry Wait Until Page Not Contains Element  ${repo_list_spinner}
+    \    Exit For Loop If  '${out[0]}'=='PASS'
+    Should Be Equal As Strings  '${out[0]}'  'PASS'
+
 Go Into Repo
     [Arguments]  ${repoName}
     Sleep  2
@@ -206,13 +218,12 @@ Go Into Repo
     \    Retry Clear Element Text  ${repo_search_input}
     \    Retry Text Input  ${repo_search_input}  ${repoName}
     \    ${out}  Run Keyword And Ignore Error  Retry Wait Until Page Contains Element  ${repo_name_element}
-    \    Exit For Loop If  '${out[0]}'=='PASS'
-    \    Capture Page Screenshot  gointo_${repoName}.png
     \    Sleep  2
-    Retry Double Keywords When Error  Retry Element Click  ${repo_name_element}  Retry Wait Until Page Not Contains Element  ${repo_name_element}
+    \    Continue For Loop If  '${out[0]}'=='FAIL'
-    Retry Wait Element  ${tag_table_column_vulnerabilities}
+    \    ${out}  Retry Click Repo Name  ${repo_name_element}
-    Retry Wait Element  ${tag_table_column_size}
+    \    Sleep  2
-    Capture Page Screenshot  gointo_${repoName}.png
+    \    Exit For Loop
+
 
 Click Index Achieve
     [Arguments]  ${tag_name}

tests/robot-cases/Group0-BAT/API_DB.robot

@@ -17,93 +17,123 @@ ${SERVER_API_ENDPOINT}  ${SERVER_URL}/api
 &{SERVER_CONFIG}  endpoint=${SERVER_API_ENDPOINT}  verify_ssl=False
 
 *** Test Cases ***
+
 Test Case - Garbage Collection
     [Tags]  gc
     Harbor API Test  ./tests/apitests/python/test_garbage_collection.py
+
 Test Case - Add Private Project Member and Check User Can See It
     [Tags]  private_member
     Harbor API Test  ./tests/apitests/python/test_add_member_to_private_project.py
+
 Test Case - Delete a Repository of a Certain Project Created by Normal User
     [Tags]  del_repo
     Harbor API Test  ./tests/apitests/python/test_del_repo.py
+
 Test Case - Add a System Global Label to a Certain Tag
     [Tags]  global_lbl
     Harbor API Test  ./tests/apitests/python/test_add_sys_label_to_tag.py
+
 Test Case - Add Replication Rule
     [Tags]  replic_rule
     Harbor API Test  ./tests/apitests/python/test_add_replication_rule.py
+
 Test Case - Edit Project Creation
     [Tags]  pro_creation
     Harbor API Test  ./tests/apitests/python/test_edit_project_creation.py
+
 Test Case - Manage Project Member
     [Tags]  member
     Harbor API Test  ./tests/apitests/python/test_manage_project_member.py
+
 Test Case - Project Level Policy Content Trust
     [Tags]  content_trust
     Harbor API Test  ./tests/apitests/python/test_project_level_policy_content_trust.py
+
 Test Case - User View Logs
     [Tags]  view_logs
     Harbor API Test  ./tests/apitests/python/test_user_view_logs.py
+
 Test Case - List Helm Charts
     [Tags]  list_helm_charts
     Harbor API Test  ./tests/apitests/python/test_list_helm_charts.py
+
 Test Case - Assign Sys Admin
     [Tags]  assign_adin
     Harbor API Test  ./tests/apitests/python/test_assign_sys_admin.py
+
 Test Case - Copy Artifact Outside Project
     [Tags]  copy_artifact
     Harbor API Test  ./tests/apitests/python/test_copy_artifact_outside_project.py
+
 Test Case - Robot Account
     [Tags]  robot_account
     Harbor API Test  ./tests/apitests/python/test_robot_account.py
+
 Test Case - Sign A Image
     [Tags]  sign_image
     Harbor API Test  ./tests/apitests/python/test_sign_image.py
+
 Test Case - Project Quota
     [Tags]  quota
     Harbor API Test  ./tests/apitests/python/test_project_quota.py
+
 Test Case - System Level CVE Whitelist
     [Tags]  sys_cve
     Harbor API Test  ./tests/apitests/python/test_sys_cve_whitelists.py
+
 Test Case - Project Level CVE Whitelist
     [Tags]  pro_cve
     Harbor API Test  ./tests/apitests/python/test_project_level_cve_whitelist.py
+
 Test Case - Tag Retention
     [Tags]  tag_retention
     Harbor API Test  ./tests/apitests/python/test_retention.py
+
 Test Case - Health Check
     [Tags]  health
     Harbor API Test  ./tests/apitests/python/test_health_check.py
+
 Test Case - Push Index By Docker Manifest
     [Tags]  push_index
     Harbor API Test  ./tests/apitests/python/test_push_index_by_docker_manifest.py
+
 Test Case - Push Chart By Helm3 Chart CLI
     [Tags]  push_chart
     Harbor API Test  ./tests/apitests/python/test_push_chart_by_helm3_chart_cli.py
+
 Test Case - Push Cnab Bundle
     [Tags]  push_cnab
     Harbor API Test  ./tests/apitests/python/test_push_cnab_bundle.py
+
 Test Case - Create/Delete tag
     [Tags]  tag_cuid
     Harbor API Test  ./tests/apitests/python/test_create_delete_tag.py
+
 Test Case - Scan Image
     [Tags]  scan
     Harbor API Test  ./tests/apitests/python/test_scan_image_artifact.py
+
 Test Case - Scan All Images
     [Tags]  scan_all
     Harbor API Test  ./tests/apitests/python/test_system_level_scan_all.py
+
 Test Case - Registry API
     [Tags]  reg_api
     Harbor API Test  ./tests/apitests/python/test_registry_api.py
+
 Test Case - Push Image With Special Name
     [Tags]  special_repo_name
     Harbor API Test  ./tests/apitests/python/test_push_image_with_special_name.py
+
 Test Case - Push Artifact With ORAS CLI
     [Tags]  oras
     Harbor API Test  ./tests/apitests/python/test_push_files_by_oras.py
+
 Test Case - Push Singularity file With Singularity CLI
     [Tags]  singularity
     Harbor API Test  ./tests/apitests/python/test_push_sif_by_singularity.py
+
 Test Case - Push Chart File To Chart Repository By Helm V2 With Robot Account
     [Tags]  helm2
     Harbor API Test  ./tests/apitests/python/test_push_chart_by_helm2_helm3_with_robot_Account.py

tests/robot-cases/Group1-Nightly/Common.robot

@@ -526,11 +526,8 @@ Test Case - Tag CRUD
     Should Contain Tag   latest
     # add more than one tag
     Add A New Tag   123
-    Should Contain Tag   123
     Add A New Tag   456
-    Should Contain Tag   456
+    Delete A Tag  latest
-    Delete A Tag   latest
-    Should Not Contain Tag   latest
     Close Browser
 
 Test Case - Tag Retention