Merge pull request #14935 from danfengliu/fix-build-base-workflow-dev-issue

Fix build base workflow issues

.github/workflows/build-package.yml

@@ -49,7 +49,12 @@ jobs:
         if: contains(steps.changed-files.outputs.modified, 'Dockerfile.base') || contains(steps.changed-files.outputs.modified, 'VERSION')
         run: |
           set -x
-          base_image_tag=$(cat ./VERSION)
+          target_branch="$(echo ${GITHUB_REF#refs/heads/})"
+          if [[ $target_branch == "release-"* ]]; then
+            base_image_tag=$(cat ./VERSION)
+          else
+            base_image_tag=dev
+          fi
           cd src/github.com/goharbor/harbor
           sudo make build_base_docker -e BASEIMAGETAG=$base_image_tag -e REGISTRYUSER="${{ secrets.DOCKER_HUB_USERNAME }}" -e REGISTRYPASSWORD="${{ secrets.DOCKER_HUB_PASSWORD }}" -e PUSHBASEIMAGE=yes
         continue-on-error: true

Makefile

@@ -421,6 +421,12 @@ build_standalone_db_migrator: compile_standalone_db_migrator
 	make -f $(MAKEFILEPATH_PHOTON)/Makefile _build_standalone_db_migrator -e BASEIMAGETAG=$(BASEIMAGETAG) -e VERSIONTAG=$(VERSIONTAG)
 
 build_base_docker:
+	if [ -n "$(REGISTRYUSER)" ] && [ -n "$(REGISTRYPASSWORD)" ] ; then \
+		docker login -u $(REGISTRYUSER) -p $(REGISTRYPASSWORD) ; \
+	else \
+		echo "No docker credentials provided, please make sure enough priviledges to access docker hub!" ; \
+	fi
+
 	@for name in $(BUILDBASETARGET); do \
 		echo $$name ; \
 		sleep 30 ; \

tests/apitests/python/library/base.py

@@ -31,7 +31,7 @@ def _create_client(server, credential, debug, api_type="products"):
     cfg = None
     if api_type in ('projectv2', 'artifact', 'repository', 'scanner', 'scan', 'scanall', 'preheat', 'quota',
                     'replication', 'registry', 'robot', 'gc', 'retention', 'immutable', 'system_cve_allowlist',
-                    'configure', 'user', 'member', 'health', 'label'):
+                    'configure', 'user', 'member', 'health', 'label', 'webhook'):
         cfg = v2_swagger_client.Configuration()
     else:
         cfg = swagger_client.Configuration()
@@ -54,12 +54,12 @@ def _create_client(server, credential, debug, api_type="products"):
         cfg.auth_settings = types.MethodType(lambda self: {}, cfg)
 
     return {
-        "chart":   client.ChartRepositoryApi(client.ApiClient(cfg)),
+        "chart": client.ChartRepositoryApi(client.ApiClient(cfg)),
-        "products":   swagger_client.ProductsApi(swagger_client.ApiClient(cfg)),
+        "products": swagger_client.ProductsApi(swagger_client.ApiClient(cfg)),
-        "projectv2":  v2_swagger_client.ProjectApi(v2_swagger_client.ApiClient(cfg)),
+        "projectv2":v2_swagger_client.ProjectApi(v2_swagger_client.ApiClient(cfg)),
-        "artifact":   v2_swagger_client.ArtifactApi(v2_swagger_client.ApiClient(cfg)),
+        "artifact": v2_swagger_client.ArtifactApi(v2_swagger_client.ApiClient(cfg)),
-        "preheat":   v2_swagger_client.PreheatApi(v2_swagger_client.ApiClient(cfg)),
+        "preheat": v2_swagger_client.PreheatApi(v2_swagger_client.ApiClient(cfg)),
-        "quota":   v2_swagger_client.QuotaApi(v2_swagger_client.ApiClient(cfg)),
+        "quota": v2_swagger_client.QuotaApi(v2_swagger_client.ApiClient(cfg)),
         "repository": v2_swagger_client.RepositoryApi(v2_swagger_client.ApiClient(cfg)),
         "scan": v2_swagger_client.ScanApi(v2_swagger_client.ApiClient(cfg)),
         "scanall": v2_swagger_client.ScanAllApi(v2_swagger_client.ApiClient(cfg)),
@@ -67,15 +67,16 @@ def _create_client(server, credential, debug, api_type="products"):
         "replication": v2_swagger_client.ReplicationApi(v2_swagger_client.ApiClient(cfg)),
         "registry": v2_swagger_client.RegistryApi(v2_swagger_client.ApiClient(cfg)),
         "robot": v2_swagger_client.RobotApi(v2_swagger_client.ApiClient(cfg)),
-        "gc":   v2_swagger_client.GcApi(v2_swagger_client.ApiClient(cfg)),
+        "gc": v2_swagger_client.GcApi(v2_swagger_client.ApiClient(cfg)),
-        "retention":   v2_swagger_client.RetentionApi(v2_swagger_client.ApiClient(cfg)),
+        "retention": v2_swagger_client.RetentionApi(v2_swagger_client.ApiClient(cfg)),
-        "immutable":   v2_swagger_client.ImmutableApi(v2_swagger_client.ApiClient(cfg)),
+        "immutable": v2_swagger_client.ImmutableApi(v2_swagger_client.ApiClient(cfg)),
-        "system_cve_allowlist":  v2_swagger_client.SystemCVEAllowlistApi(v2_swagger_client.ApiClient(cfg)),
+        "system_cve_allowlist": v2_swagger_client.SystemCVEAllowlistApi(v2_swagger_client.ApiClient(cfg)),
-        "configure":   v2_swagger_client.ConfigureApi(v2_swagger_client.ApiClient(cfg)),
+        "configure": v2_swagger_client.ConfigureApi(v2_swagger_client.ApiClient(cfg)),
-        "label":   v2_swagger_client.LabelApi(v2_swagger_client.ApiClient(cfg)),
+        "label": v2_swagger_client.LabelApi(v2_swagger_client.ApiClient(cfg)),
         "user": v2_swagger_client.UserApi(v2_swagger_client.ApiClient(cfg)),
         "member": v2_swagger_client.MemberApi(v2_swagger_client.ApiClient(cfg)),
-        "health":   v2_swagger_client.HealthApi(v2_swagger_client.ApiClient(cfg)),
+        "health": v2_swagger_client.HealthApi(v2_swagger_client.ApiClient(cfg)),
+        "webhook": v2_swagger_client.WebhookApi(v2_swagger_client.ApiClient(cfg))
     }.get(api_type,'Error: Wrong API type')
 
 def _assert_status_code(expect_code, return_code, err_msg = r"HTTPS status code s not as we expected. Expected {}, while actual HTTPS status code is {}."):

tests/apitests/python/library/webhook.py

@@ -0,0 +1,100 @@
+# -*- coding: utf-8 -*-
+
+import time
+import base
+import v2_swagger_client
+from v2_swagger_client.rest import ApiException
+
+class Webhook(base.Base):
+    def __init__(self):
+        super(Webhook,self).__init__(api_type="webhook")
+
+    def create_webhook(self, project_id, targets, event_types = ["DELETE_ARTIFACT",
+            "PULL_ARTIFACT",
+            "PUSH_ARTIFACT",
+            "DELETE_CHART",
+            "DOWNLOAD_CHART",
+            "UPLOAD_CHART","QUOTA_EXCEED",
+            "QUOTA_WARNING","SCANNING_FAILED",
+            "TAG_RETENTION"],
+            name = None, desc = None, enabled = True,
+            expect_status_code = 201, expect_response_body = None, **kwargs):
+
+        if name is None:
+            name = base._random_name("webhook") + (str(project_id))
+        if desc is None:
+            desc = base._random_name("webhook desc") + (str(project_id))
+        policy = v2_swagger_client.WebhookPolicy(
+            name = name,
+            description = desc,
+            project_id = project_id,
+            targets = targets,
+            event_types = event_types,
+            enabled = enabled
+        )
+
+        try:
+            _, status_code, header = self._get_client(**kwargs).create_webhook_policy_of_project_with_http_info(project_id, policy)
+        except ApiException as e:
+            base._assert_status_code(expect_status_code, e.status)
+            if expect_response_body is not None:
+                base._assert_status_body(expect_response_body, e.body)
+            return
+        base._assert_status_code(expect_status_code, status_code)
+        base._assert_status_code(201, status_code)
+        return base._get_id_from_header(header), name
+
+    def update_webhook(self, project_id, webhook_policy_id, event_types = None,
+            name = None, desc = None, enabled = None, targets = None,
+            expect_status_code=200, expect_response_body=None, **kwargs):
+
+        policy = v2_swagger_client.WebhookPolicy()
+        if name is not None:
+            policy.name = name
+        if desc is not None:
+            policy.desc = desc
+        if enabled is not None:
+            policy.enabled = enabled
+        if targets is not None:
+            policy.targets = targets
+        if event_types is not None:
+            policy.event_types = event_types
+
+        try:
+            _, status_code, header = self._get_client(**kwargs).update_webhook_policy_of_project_with_http_info(project_id, webhook_policy_id, policy)
+        except ApiException as e:
+            base._assert_status_code(expect_status_code, e.status)
+            if expect_response_body is not None:
+                base._assert_status_body(expect_response_body, e.body)
+            return
+        base._assert_status_code(expect_status_code, status_code)
+        base._assert_status_code(200, status_code)
+
+    def get_webhook(self, project_id, webhook_policy_id, expect_status_code=200, expect_response_body=None, **kwargs):
+
+        try:
+            data , status_code, header = self._get_client(**kwargs).get_webhook_policy_of_project_with_http_info(project_id, webhook_policy_id)
+        except ApiException as e:
+            base._assert_status_code(expect_status_code, e.status)
+            if expect_response_body is not None:
+                base._assert_status_body(expect_response_body, e.body)
+            return
+        base._assert_status_code(expect_status_code, status_code)
+        base._assert_status_code(200, status_code)
+        print("Webhooks:", data)
+        return data
+
+    def delete_webhook(self, project_id, webhook_policy_id, expect_status_code=200, expect_response_body=None, **kwargs):
+
+        try:
+            _ , status_code, _ = self._get_client(**kwargs).delete_webhook_policy_of_project_with_http_info(project_id, webhook_policy_id)
+        except ApiException as e:
+            base._assert_status_code(expect_status_code, e.status)
+            if expect_response_body is not None:
+                base._assert_status_body(expect_response_body, e.body)
+            return
+        base._assert_status_code(expect_status_code, status_code)
+        base._assert_status_code(200, status_code)
+
+
+

tests/apitests/python/test_webhook_crud.py

@@ -0,0 +1,84 @@
+from __future__ import absolute_import
+
+
+import unittest
+import v2_swagger_client
+from testutils import ADMIN_CLIENT, suppress_urllib3_warning
+from testutils import harbor_server
+from testutils import TEARDOWN
+from library.base import _assert_status_code
+from library.project import Project
+from library.user import User
+from library.repository import Repository
+from library.repository import push_self_build_image_to_project
+from library.webhook import Webhook
+
+class TestProjects(unittest.TestCase):
+    @suppress_urllib3_warning
+    def setUp(self):
+        self.url = ADMIN_CLIENT["endpoint"]
+        self.user_password = "Aa123456"
+        self.project= Project()
+        self.user= User()
+        self.webhook= Webhook()
+        self.user_id, self.project_id = [None] * 2
+        self.user_id, self.user_name = self.user.create_user(user_password = self.user_password, **ADMIN_CLIENT)
+        self.USER_CLIENT = dict(with_signature = True, with_immutable_status = True, endpoint = self.url, username = self.user_name, password = self.user_password)
+
+    @unittest.skipIf(TEARDOWN == True, "Test data won't be erased.")
+    def tearDown(self):
+        #1. Delete project(PA);
+        self.project.delete_project(self.project_id, **self.USER_CLIENT)
+
+        #2. Delete user(UA).
+        self.user.delete_user(self.user_id, **ADMIN_CLIENT)
+
+    def testDelRepo(self):
+        """
+        Test case:
+            Webhook CRUD
+        Test step and expected result:
+            1. Create a new user(UA);
+            2. Create a new project(PA) by user(UA);
+            3. Create a new webhook(WA) in project(PA) by user(UA);
+            4. Modify properties of webhook(WA), it should be successful;
+            5. Delete webhook(WA) by user(UA), it should be successful.
+        Tear down:
+            1. Delete project(PA);
+            2. Delete user(UA).
+        """
+        #2. Create a new project;
+        self.project_id, project_name = self.project.create_project(metadata = {"public": "false"}, **self.USER_CLIENT)
+        print("project_id:",self.project_id)
+        print("project_name:",project_name)
+
+        target_1 = v2_swagger_client.WebhookTargetObject(
+            address = "https://hooks.slack.com/services",
+            skip_cert_verify = False,
+            type = "slack",
+            auth_header = "aaa"
+        )
+        target_2 = v2_swagger_client.WebhookTargetObject(
+            address = "https://202.10.12.13",
+            skip_cert_verify = False,
+            type = "http",
+            auth_header = "aaa"
+        )
+        #This need to be removed once issue #13378 fixed.
+        policy_id, policy_name = self.webhook.create_webhook(self.project_id, [target_1, target_2], **self.USER_CLIENT)
+        target_1 = v2_swagger_client.WebhookTargetObject(
+            address = "https://hooks.slack.com/services/new",
+            skip_cert_verify = True,
+            type = "http",
+            auth_header = "bbb"
+        )
+        self.webhook.get_webhook(self.project_id, policy_id, **self.USER_CLIENT)
+        self.webhook.update_webhook(self.project_id, policy_id, name = "new_name",auth_header = "new_header",
+            event_types = ["DELETE_ARTIFACT", "TAG_RETENTION"], enabled = False, targets = [target_1], **self.USER_CLIENT)
+        self.webhook.get_webhook(self.project_id, policy_id, **self.USER_CLIENT)
+
+        self.webhook.delete_webhook(self.project_id, policy_id, **self.USER_CLIENT)
+        self.webhook.get_webhook(self.project_id, policy_id, expect_status_code = 404, **self.USER_CLIENT)
+if __name__ == '__main__':
+    unittest.main()
+

tests/robot-cases/Group0-BAT/API_DB.robot

@@ -160,3 +160,7 @@ Test Case - Metrics
 Test Case - Project Level Policy Content Trust
     [Tags]  content_trust
     Harbor API Test  ./tests/apitests/python/test_project_level_policy_content_trust.py
+
+Test Case - Webhook CRUD
+    [Tags]  webhook
+    Harbor API Test  ./tests/apitests/python/test_webhook_crud.py

tests/robot-cases/Group3-Upgrade/feature_map.json

@@ -491,6 +491,32 @@
             "version":"2.2"
         }
     ],
+    "update_systemsetting":[
+        {
+            "branch":1,
+            "version":"1.8"
+        },
+        {
+            "branch":1,
+            "version":"1.9"
+        },
+        {
+            "branch":1,
+            "version":"1.10"
+        },
+        {
+            "branch":1,
+            "version":"2.0"
+        },
+        {
+            "branch":1,
+            "version":"2.1"
+        },
+        {
+            "branch":2,
+            "version":"2.2"
+        }
+    ],
     "populate_quotas":[
         {
             "branch":1,

tests/robot-cases/Group3-Upgrade/prepare.py

@@ -316,7 +316,10 @@ class HarborAPI:
         body=dict(body=payload)
         request(url+"system/scanAll/schedule", 'post', **body)
 
-    def update_systemsetting(self, emailfrom, emailhost, emailport, emailuser, creation, selfreg, token, robot_token):
+    @get_feature_branch
+    def update_systemsetting(self, emailfrom, emailhost, emailport, emailuser, creation, selfreg, token, robot_token, **kwargs):
+        if kwargs["branch"] == 1:
+            robot_token = float(robot_token)*60*24
         payload = {
             "auth_mode": "db_auth",
             "email_from": emailfrom,
@@ -575,7 +578,7 @@ class HarborAPI:
             except Exception as e:
                 print(str(e))
                 pass
-        open(target, 'wb').write(ca_content.encode('utf-8'))
+        open(target, 'wb').write(str(ca_content).encode('utf-8'))
 
     @get_feature_branch
     def push_artifact_index(self, project, name, tag, **kwargs):
@@ -668,7 +671,7 @@ def do_data_creation():
                                    data["configuration"]["projectcreation"],
                                    data["configuration"]["selfreg"],
                                    float(data["configuration"]["token"]),
-                                   float(data["configuration"]["robot_token"])*60*24)
+                                   float(data["configuration"]["robot_token"]), version=args.version)
 
     harborAPI.add_sys_allowlist(data["configuration"]["deployment_security"], version=args.version)