Merge pull request #13437 from ywk253100/200929_replication

Refactor the replication execution

api/v2.0/legacy_swagger.yaml

@@ -738,231 +738,6 @@ paths:
           description: The auth mode of the system is not "oidc_auth", or the user is not onboarded via OIDC AuthN.
         '500':
           description: Unexpected internal errors.
-  /replication/executions:
-    get:
-      summary: List replication executions.
-      description: |
-        This endpoint let user list replication executions.
-      parameters:
-        - name: policy_id
-          in: query
-          type: integer
-          required: false
-          description: The policy ID.
-        - name: status
-          in: query
-          type: string
-          required: false
-          description: The execution status.
-        - name: trigger
-          in: query
-          type: string
-          required: false
-          description: The trigger mode.
-        - name: page
-          in: query
-          type: integer
-          required: false
-          description: The page.
-        - name: page_size
-          in: query
-          type: integer
-          required: false
-          description: The page size.
-      tags:
-        - Products
-      responses:
-        '200':
-          description: Success
-          schema:
-            type: array
-            items:
-              $ref: '#/definitions/ReplicationExecution'
-          headers:
-            X-Total-Count:
-              description: The total count of available items
-              type: integer
-            Link:
-              description: Link to previous page and next page
-              type: string
-        '401':
-          description: User need to login first.
-        '403':
-          description: User has no privilege for the operation.
-        '500':
-          description: Unexpected internal errors.
-    post:
-      summary: Start one execution of the replication.
-      description: |
-        This endpoint is for user to start one execution of the replication.
-      parameters:
-        - name: execution
-          in: body
-          description: The execution that needs to be started, only the property "policy_id" is needed.
-          required: true
-          schema:
-            $ref: '#/definitions/ReplicationExecution'
-      tags:
-        - Products
-      responses:
-        '201':
-          description: Success.
-          headers:
-            Location:
-              type: string
-              description: The URL of the created resource
-        '400':
-          description: Bad request.
-        '401':
-          description: User need to login first.
-        '403':
-          description: User has no privilege for the operation.
-        '415':
-          $ref: '#/responses/UnsupportedMediaType'
-        '500':
-          description: Unexpected internal errors.
-  /replication/executions/{id}:
-    get:
-      summary: Get the execution of the replication.
-      description: |
-        This endpoint is for user to get one execution of the replication.
-      parameters:
-        - name: id
-          in: path
-          type: integer
-          format: int64
-          description: The execution ID.
-          required: true
-      tags:
-        - Products
-      responses:
-        '200':
-          description: Success.
-          schema:
-            $ref: '#/definitions/ReplicationExecution'
-        '400':
-          description: Bad request.
-        '401':
-          description: User need to login first.
-        '403':
-          description: User has no privilege for the operation.
-        '404':
-          description: Resource requested does not exist.
-        '415':
-          $ref: '#/responses/UnsupportedMediaType'
-        '500':
-          description: Unexpected internal errors.
-    put:
-      summary: Stop the execution of the replication.
-      description: |
-        This endpoint is for user to stop one execution of the replication.
-      parameters:
-        - name: id
-          in: path
-          type: integer
-          format: int64
-          description: The execution ID.
-          required: true
-      tags:
-        - Products
-      responses:
-        '200':
-          description: Success.
-        '400':
-          description: Bad request.
-        '401':
-          description: User need to login first.
-        '403':
-          description: User has no privilege for the operation.
-        '404':
-          description: Resource requested does not exist.
-        '415':
-          $ref: '#/responses/UnsupportedMediaType'
-        '500':
-          description: Unexpected internal errors.
-  /replication/executions/{id}/tasks:
-    get:
-      summary: Get the task list of one execution.
-      description: |
-        This endpoint is for user to get the task list of one execution.
-      parameters:
-        - name: id
-          in: path
-          type: integer
-          format: int64
-          description: The execution ID.
-          required: true
-        - name: page
-          in: query
-          type: integer
-          format: int32
-          required: false
-          description: The page number.
-        - name: page_size
-          in: query
-          type: integer
-          format: int32
-          required: false
-          description: The size of per page.
-      tags:
-        - Products
-      responses:
-        '200':
-          description: Success.
-          schema:
-            type: array
-            items:
-              $ref: '#/definitions/ReplicationTask'
-          headers:
-            X-Total-Count:
-              description: The total count of available items
-              type: integer
-            Link:
-              description: Link to previous page and next page
-              type: string
-        '400':
-          description: Bad request.
-        '401':
-          description: User need to login first.
-        '403':
-          description: User has no privilege for the operation.
-        '404':
-          description: Resource requested does not exist.
-        '500':
-          description: Unexpected internal errors.
-  /replication/executions/{id}/tasks/{task_id}/log:
-    get:
-      summary: Get the log of one task.
-      description: |
-        This endpoint is for user to get the log of one task.
-      parameters:
-        - name: id
-          in: path
-          type: integer
-          format: int64
-          description: The execution ID.
-          required: true
-        - name: task_id
-          in: path
-          type: integer
-          format: int64
-          description: The task ID.
-          required: true
-      tags:
-        - Products
-      responses:
-        '200':
-          description: Success.
-        '400':
-          description: Bad request.
-        '401':
-          description: User need to login first.
-        '403':
-          description: User has no privilege for the operation.
-        '404':
-          description: Resource requested does not exist.
-        '500':
-          description: Unexpected internal errors.
   /replication/policies:
     get:
       summary: List replication policies
@@ -4972,77 +4747,6 @@ definitions:
         description: The filter values
         items:
           type: string
-  ReplicationExecution:
-    type: object
-    description: The replication execution
-    properties:
-      id:
-        type: integer
-        description: The ID
-      policy_id:
-        type: integer
-        description: The policy ID
-      status:
-        type: string
-        description: The status
-      status_text:
-        type: string
-        description: The status text
-      trigger:
-        type: string
-        description: The trigger mode
-      total:
-        type: integer
-        description: The total count of all tasks
-      failed:
-        type: integer
-        description: The count of failed tasks
-      succeed:
-        type: integer
-        description: The count of succeed tasks
-      in_progress:
-        type: integer
-        description: The count of in_progress tasks
-      stopped:
-        type: integer
-        description: The count of stopped tasks
-      start_time:
-        type: string
-        description: The start time
-      end_time:
-        type: string
-        description: The end time
-  ReplicationTask:
-    type: object
-    description: The replication task
-    properties:
-      id:
-        type: integer
-        description: The ID
-      execution_id:
-        type: integer
-        description: The execution ID
-      resource_type:
-        type: string
-        description: The resource type
-      src_resource:
-        type: string
-        description: The source resource
-      dst_resource:
-        type: string
-        description: The destination resource
-      job_id:
-        type: string
-        description: The job ID
-      status:
-        type: string
-        description: The status
-      start_time:
-        type: string
-        description: The start time
-      end_time:
-        type: string
-        description: The end time
   Namespace:
     type: object
     description: The namespace of registry

api/v2.0/swagger.yaml

@@ -1431,6 +1431,209 @@ paths:
           $ref: '#/responses/404'
         '500':
           $ref: '#/responses/500'
+  /replication/executions:
+    get:
+      summary: List replication executions
+      description: List replication executions
+      tags:
+        - replication
+      operationId: listReplicationExecutions
+      parameters:
+        - $ref: '#/parameters/page'
+        - $ref: '#/parameters/pageSize'
+        - name: policy_id
+          in: query
+          type: integer
+          required: false
+          description: The ID of the policy that the executions belong to.
+        - name: status
+          in: query
+          type: string
+          required: false
+          description: The execution status.
+        - name: trigger
+          in: query
+          type: string
+          required: false
+          description: The trigger mode.
+      responses:
+        '200':
+          description: Success
+          headers:
+            X-Total-Count:
+              description: The total count of the resources
+              type: integer
+            Link:
+              description: Link refers to the previous page and next page
+              type: string
+          schema:
+            type: array
+            items:
+              $ref: '#/definitions/ReplicationExecution'
+        '401':
+          $ref: '#/responses/401'
+        '403':
+          $ref: '#/responses/403'
+        '500':
+          $ref: '#/responses/500'
+    post:
+      summary: Start one replication execution
+      description: Start one replication execution according to the policy
+      tags:
+        - replication
+      operationId: startReplication
+      parameters:
+        - name: execution
+          in: body
+          description: The ID of policy that the execution belongs to
+          required: true
+          schema:
+            $ref: '#/definitions/StartReplicationExecution'
+      responses:
+        '201':
+          $ref: '#/responses/201'
+        '400':
+          $ref: '#/responses/400'
+        '401':
+          $ref: '#/responses/401'
+        '403':
+          $ref: '#/responses/403'
+        '500':
+          $ref: '#/responses/500'
+  /replication/executions/{id}:
+    get:
+      summary: Get the specific replication execution
+      description: Get the replication execution specified by ID
+      tags:
+        - replication
+      operationId: getReplicationExecution
+      parameters:
+        - name: id
+          in: path
+          type: integer
+          format: int64
+          description: The ID of the execution.
+          required: true
+      responses:
+        '200':
+          description: Success
+          schema:
+            $ref: '#/definitions/ReplicationExecution'
+        '401':
+          $ref: '#/responses/401'
+        '403':
+          $ref: '#/responses/403'
+        '404':
+          $ref: '#/responses/404'
+        '500':
+          $ref: '#/responses/500'
+    put:
+      summary: Stop the specific replication execution
+      description: Stop the replication execution specified by ID
+      tags:
+        - replication
+      operationId: stopReplication
+      parameters:
+        - name: id
+          in: path
+          type: integer
+          format: int64
+          description: The ID of the execution.
+          required: true
+      responses:
+        '200':
+          $ref: '#/responses/200'
+        '401':
+          $ref: '#/responses/401'
+        '403':
+          $ref: '#/responses/403'
+        '404':
+          $ref: '#/responses/404'
+        '500':
+          $ref: '#/responses/500'
+  /replication/executions/{id}/tasks:
+    get:
+      summary: List replication tasks for a specific execution
+      description: List replication tasks for a specific execution
+      tags:
+        - replication
+      operationId: listReplicationTasks
+      parameters:
+        - $ref: '#/parameters/page'
+        - $ref: '#/parameters/pageSize'
+        - name: id
+          in: path
+          type: integer
+          format: int64
+          description: The ID of the execution that the tasks belongs to.
+          required: true
+        - name: status
+          in: query
+          type: string
+          required: false
+          description: The task status.
+        - name: resource_type
+          in: query
+          type: string
+          required: false
+          description: The resource type.
+      responses:
+        '200':
+          description: Success
+          headers:
+            X-Total-Count:
+              description: The total count of the resources
+              type: integer
+            Link:
+              description: Link refers to the previous page and next page
+              type: string
+          schema:
+            type: array
+            items:
+              $ref: '#/definitions/ReplicationTask'
+        '401':
+          $ref: '#/responses/401'
+        '403':
+          $ref: '#/responses/403'
+        '500':
+          $ref: '#/responses/500'
+  /replication/executions/{id}/tasks/{task_id}/log:
+    get:
+      summary: Get the log of the specific replication task
+      description: Get the log of the specific replication task
+      tags:
+        - replication
+      operationId: getReplicationLog
+      parameters:
+        - name: id
+          in: path
+          type: integer
+          format: int64
+          description: The ID of the execution that the tasks belongs to.
+          required: true
+        - name: task_id
+          in: path
+          type: integer
+          format: int64
+          description: The ID of the task.
+          required: true
+      responses:
+        '200':
+          description: Success
+          headers:
+            Content-Type:
+              description: The content type of response body
+              type: string
+          schema:
+            type: string
+        '401':
+          $ref: '#/responses/401'
+        '403':
+          $ref: '#/responses/403'
+        '404':
+          $ref: '#/responses/404'
+        '500':
+          $ref: '#/responses/500'
 parameters:
   query:
     name: q
@@ -2123,6 +2326,7 @@ definitions:
         description: The status message of task
       run_count:
         type: integer
+        format: int32
         description: The count of task run
       extra_attrs:
         $ref: '#/definitions/ExtraAttrs'
@@ -2398,3 +2602,94 @@ definitions:
     additionalProperties:
       type: integer
       format: int64
+  ReplicationExecution:
+    type: object
+    description: The replication execution
+    properties:
+      id:
+        type: integer
+        description: The ID of the execution
+      policy_id:
+        type: integer
+        description: The ID if the policy that the execution belongs to
+      status:
+        type: string
+        description: The status of the execution
+      trigger:
+        type: string
+        description: The trigger mode
+      start_time:
+        type: string
+        format: date-time
+        description: The start time
+      end_time:
+        type: string
+        format: date-time
+        description: The end time
+      status_text:
+        type: string
+        x-omitempty: false
+        description: The status text
+      total:
+        type: integer
+        x-omitempty: false
+        description: The total count of all executions
+      failed:
+        type: integer
+        x-omitempty: false
+        description: The count of failed executions
+      succeed:
+        type: integer
+        x-omitempty: false
+        description: The count of succeed executions
+      in_progress:
+        type: integer
+        x-omitempty: false
+        description: The count of in_progress executions
+      stopped:
+        type: integer
+        x-omitempty: false
+        description: The count of stopped executions
+  StartReplicationExecution:
+    type: object
+    properties:
+      policy_id:
+        type: integer
+        format: int64
+        description: The ID of policy that the execution belongs to.
+  ReplicationTask:
+    type: object
+    description: The replication task
+    properties:
+      id:
+        type: integer
+        description: The ID of the task
+      execution_id:
+        type: integer
+        description: The ID of the execution that the task belongs to
+      status:
+        type: string
+        description: The status of the task
+      job_id:
+        type: string
+        description: The ID of the underlying job that the task related to
+      operation:
+        type: string
+        description: The operation of the task
+      resource_type:
+        type: string
+        description: The type of the resource that the task operates
+      src_resource:
+        type: string
+        description: The source resource that the task operates
+      dst_resource:
+        type: string
+        description: The destination resource that the task operates
+      start_time:
+        type: string
+        format: date-time
+        description: The start time of the task
+      end_time:
+        type: string
+        format: date-time
+        description: The end time of the task

make/migrations/postgresql/0050_2.2.0_schema.up.sql

@@ -8,6 +8,10 @@ UPDATE role SET role_id=5 WHERE name='limitedGuest' AND role_id!=5;
 ALTER TABLE schedule ADD COLUMN IF NOT EXISTS cron_type varchar(64);
 ALTER TABLE robot ADD COLUMN IF NOT EXISTS secret varchar(2048);
 
+ALTER TABLE task ADD COLUMN IF NOT EXISTS vendor_type varchar(16);
+UPDATE task SET vendor_type = execution.vendor_type FROM execution WHERE task.execution_id = execution.id;
+ALTER TABLE task ALTER COLUMN vendor_type SET NOT NULL;
+
 DO $$
 DECLARE
     art RECORD;
@@ -46,3 +50,193 @@ CREATE TABLE  IF NOT EXISTS permission_policy (
  creation_time timestamp default CURRENT_TIMESTAMP,
  CONSTRAINT unique_rbac_policy UNIQUE (scope, resource, action, effect)
 );
+
+/*delete the replication execution records whose policy doesn't exist*/
+DELETE FROM replication_execution
+    WHERE id IN (SELECT re.id FROM replication_execution re
+        LEFT JOIN replication_policy rp ON re.policy_id=rp.id
+        WHERE rp.id IS NULL);
+
+/*delete the replication task records whose execution doesn't exist*/
+DELETE FROM replication_task
+    WHERE id IN (SELECT rt.id FROM replication_task rt
+        LEFT JOIN replication_execution re ON rt.execution_id=re.id
+        WHERE re.id IS NULL);
+
+/*fill the task count, status and end_time of execution based on the tasks*/
+DO $$
+DECLARE
+    rep_exec RECORD;
+    status_count RECORD;
+    rep_status varchar(32);
+BEGIN
+    FOR rep_exec IN SELECT * FROM replication_execution
+    LOOP
+      /*the replication status is set directly in some cases, so skip if the status is a final one*/
+      IF rep_exec.status='Stopped' OR rep_exec.status='Failed' OR rep_exec.status='Succeed' THEN
+        CONTINUE;
+      END IF;
+      /*fulfill the status count*/
+      FOR status_count IN SELECT status, COUNT(*) as c FROM replication_task WHERE execution_id=rep_exec.id GROUP BY status
+      LOOP
+        IF status_count.status = 'Stopped' THEN
+          UPDATE replication_execution SET stopped=status_count.c WHERE id=rep_exec.id;
+        ELSIF status_count.status = 'Failed' THEN
+          UPDATE replication_execution SET failed=status_count.c WHERE id=rep_exec.id;
+        ELSIF status_count.status = 'Succeed' THEN
+          UPDATE replication_execution SET succeed=status_count.c WHERE id=rep_exec.id;
+        ELSE
+          UPDATE replication_execution SET in_progress=status_count.c WHERE id=rep_exec.id;
+        END IF;
+      END LOOP;
+
+      /*reload the execution record*/
+      SELECT * INTO rep_exec FROM replication_execution where id=rep_exec.id;
+
+      /*calculate the status*/
+      IF rep_exec.in_progress>0 THEN
+        rep_status = 'InProgress';
+      ELSIF rep_exec.failed>0 THEN
+        rep_status = 'Failed';
+      ELSIF rep_exec.stopped>0 THEN
+        rep_status = 'Stopped';
+      ELSE
+        rep_status = 'Succeed';
+      END IF;
+      UPDATE replication_execution SET status=rep_status WHERE id=rep_exec.id;
+
+      /*update the end time if the status is a final one*/
+      IF rep_status='Failed' OR rep_status='Stopped' OR rep_status='Succeed' THEN
+        UPDATE replication_execution
+            SET end_time=(SELECT MAX (end_time) FROM replication_task WHERE execution_id=rep_exec.id)
+            WHERE id=rep_exec.id;
+      END IF;
+    END LOOP;
+END $$;
+
+/*move the replication execution records into the new execution table*/
+ALTER TABLE replication_execution ADD COLUMN IF NOT EXISTS new_execution_id int;
+DO $$
+DECLARE
+    rep_exec RECORD;
+    trigger varchar(64);
+    status varchar(32);
+    new_exec_id integer;
+BEGIN
+    FOR rep_exec IN SELECT * FROM replication_execution
+    LOOP
+      IF rep_exec.trigger = 'scheduled' THEN
+        trigger = 'SCHEDULE';
+      ELSIF rep_exec.trigger = 'event_based' THEN
+        trigger = 'EVENT';
+      ELSE
+        trigger = 'MANUAL';
+      END IF;
+
+      IF rep_exec.status = 'InProgress' THEN
+        status = 'Running';
+      ELSIF rep_exec.status = 'Stopped' THEN
+        status = 'Stopped';
+      ELSIF rep_exec.status = 'Failed' THEN
+        status = 'Error';
+      ELSIF rep_exec.status = 'Succeed' THEN
+        status = 'Success';
+      END IF;
+      
+      INSERT INTO execution (vendor_type, vendor_id, status, status_message, revision, trigger, start_time, end_time)
+        VALUES ('REPLICATION', rep_exec.policy_id, status, rep_exec.status_text, 0, trigger, rep_exec.start_time, rep_exec.end_time) RETURNING id INTO new_exec_id;
+      UPDATE replication_execution SET new_execution_id=new_exec_id WHERE id=rep_exec.id;
+    END LOOP;
+END $$;
+
+/*move the replication task records into the new task table*/
+DO $$
+DECLARE
+    rep_task RECORD;
+    status varchar(32);
+    status_code integer;
+BEGIN
+    FOR rep_task IN SELECT * FROM replication_task
+    LOOP
+      IF rep_task.status = 'InProgress' THEN
+        status = 'Running';
+        status_code = 2;
+      ELSIF rep_task.status = 'Stopped' THEN
+        status = 'Stopped';
+        status_code = 3;
+      ELSIF rep_task.status = 'Failed' THEN
+        status = 'Error';
+        status_code = 3;
+      ELSIF rep_task.status = 'Succeed' THEN
+        status = 'Success';
+        status_code = 3;
+      ELSE
+        status = 'Pending';
+        status_code = 0;
+      END IF;
+      INSERT INTO task (vendor_type, execution_id, job_id, status, status_code, status_revision,
+        run_count, extra_attrs, creation_time, start_time, update_time, end_time)
+        VALUES ('REPLICATION', (SELECT new_execution_id FROM replication_execution WHERE id=rep_task.execution_id),
+            rep_task.job_id, status, status_code, rep_task.status_revision,
+            1, CONCAT('{"resource_type":"', rep_task.resource_type,'","source_resource":"', rep_task.src_resource, '","destination_resource":"', rep_task.dst_resource, '","operation":"', rep_task.operation,'"}')::json,
+            rep_task.start_time, rep_task.start_time, rep_task.end_time, rep_task.end_time);
+    END LOOP;
+END $$;
+
+DROP TABLE IF EXISTS replication_task;
+DROP TABLE IF EXISTS replication_execution;
+
+/*move the replication schedule job records into the new schedule table*/
+DO $$
+DECLARE
+    schd RECORD;
+    new_schd_id integer;
+    exec_id integer;
+    exec_status varchar(32);
+    task_status varchar(32);
+    task_status_code integer;
+BEGIN
+    FOR schd IN SELECT * FROM replication_schedule_job
+    LOOP
+        INSERT INTO schedule (vendor_type, vendor_id, cron, callback_func_name,
+            callback_func_param, creation_time, update_time)
+            VALUES ('REPLICATION', schd.policy_id,
+                (SELECT trigger::json->'trigger_settings'->>'cron' FROM replication_policy WHERE id=schd.policy_id),
+                'REPLICATION_CALLBACK', schd.policy_id, schd.creation_time, schd.update_time) RETURNING id INTO new_schd_id;
+        IF schd.status = 'stopped' THEN
+            exec_status = 'Stopped';
+            task_status = 'Stopped';
+            task_status_code = 3;
+        ELSIF schd.status = 'error' THEN
+            exec_status = 'Error';
+            task_status = 'Error';
+            task_status_code = 3;
+        ELSIF schd.status = 'finished' THEN
+            exec_status = 'Success';
+            task_status = 'Success';
+            task_status_code = 3;
+        ELSIF schd.status = 'running' THEN
+            exec_status = 'Running';
+            task_status = 'Running';
+            task_status_code = 2;
+        ELSEIF schd.status = 'pending' THEN
+            exec_status = 'Running';
+            task_status = 'Pending';
+            task_status_code = 0;
+        ELSEIF schd.status = 'scheduled' THEN
+            exec_status = 'Running';
+            task_status = 'Scheduled';
+            task_status_code = 1;
+        ELSE
+            exec_status = 'Running';
+            task_status = 'Pending';
+            task_status_code = 0;
+        END IF;
+        INSERT INTO execution (vendor_type, vendor_id, status, revision, trigger, start_time, end_time)
+            VALUES ('SCHEDULER', new_schd_id, exec_status, 0, 'MANUAL', schd.creation_time, schd.update_time) RETURNING id INTO exec_id;
+        INSERT INTO task (vendor_type, execution_id, job_id, status, status_code, status_revision, run_count, creation_time, start_time, update_time, end_time)
+            VALUES ('SCHEDULER',exec_id, schd.job_id, task_status, task_status_code, 0, 1, schd.creation_time, schd.creation_time, schd.update_time, schd.update_time);
+    END LOOP;
+END $$;
+
+DROP TABLE IF EXISTS replication_schedule_job;

src/controller/event/handler/init.go

@@ -1,6 +1,7 @@
 package handler
 
 import (
+	"context"
 	"github.com/goharbor/harbor/src/controller/event"
 	"github.com/goharbor/harbor/src/controller/event/handler/auditlog"
 	"github.com/goharbor/harbor/src/controller/event/handler/internal"
@@ -10,8 +11,12 @@ import (
 	"github.com/goharbor/harbor/src/controller/event/handler/webhook/chart"
 	"github.com/goharbor/harbor/src/controller/event/handler/webhook/quota"
 	"github.com/goharbor/harbor/src/controller/event/handler/webhook/scan"
+	"github.com/goharbor/harbor/src/controller/event/metadata"
+	"github.com/goharbor/harbor/src/jobservice/job"
 	"github.com/goharbor/harbor/src/lib/orm"
+	"github.com/goharbor/harbor/src/pkg/notification"
 	"github.com/goharbor/harbor/src/pkg/notifier"
+	"github.com/goharbor/harbor/src/pkg/task"
 )
 
 func init() {
@@ -54,4 +59,12 @@ func init() {
 	// internal
 	notifier.Subscribe(event.TopicPullArtifact, &internal.Handler{Context: orm.Context})
 	notifier.Subscribe(event.TopicPushArtifact, &internal.Handler{Context: orm.Context})
+
+	task.RegisterTaskStatusChangePostFunc(job.Replication, func(ctx context.Context, taskID int64, status string) error {
+		notification.AddEvent(ctx, &metadata.ReplicationMetaData{
+			ReplicationTaskID: taskID,
+			Status:            status,
+		})
+		return nil
+	})
 }

src/controller/event/handler/webhook/artifact/replication.go

@@ -10,13 +10,14 @@ import (
 	"github.com/goharbor/harbor/src/controller/event/handler/util"
 	ctlModel "github.com/goharbor/harbor/src/controller/event/model"
 	"github.com/goharbor/harbor/src/controller/project"
+	"github.com/goharbor/harbor/src/controller/replication"
 	"github.com/goharbor/harbor/src/core/config"
 	"github.com/goharbor/harbor/src/jobservice/job"
 	"github.com/goharbor/harbor/src/lib/log"
 	"github.com/goharbor/harbor/src/lib/orm"
 	"github.com/goharbor/harbor/src/pkg/notification"
 	"github.com/goharbor/harbor/src/pkg/notifier/model"
-	"github.com/goharbor/harbor/src/replication"
+	rep "github.com/goharbor/harbor/src/replication"
 	rpModel "github.com/goharbor/harbor/src/replication/model"
 )
 
@@ -71,25 +72,20 @@ func (r *ReplicationHandler) IsStateful() bool {
 }
 
 func constructReplicationPayload(event *event.ReplicationEvent) (*model.Payload, *commonModels.Project, error) {
-	task, err := replication.OperationCtl.GetTask(event.ReplicationTaskID)
+	ctx := orm.Context()
+	task, err := replication.Ctl.GetTask(ctx, event.ReplicationTaskID)
 	if err != nil {
 		log.Errorf("failed to get replication task %d: error: %v", event.ReplicationTaskID, err)
 		return nil, nil, err
 	}
-	if task == nil {
-		return nil, nil, fmt.Errorf("task %d not found with replication event", event.ReplicationTaskID)
-	}
 
-	execution, err := replication.OperationCtl.GetExecution(task.ExecutionID)
+	execution, err := replication.Ctl.GetExecution(ctx, task.ExecutionID)
 	if err != nil {
 		log.Errorf("failed to get replication execution %d: error: %v", task.ExecutionID, err)
 		return nil, nil, err
 	}
-	if execution == nil {
-		return nil, nil, fmt.Errorf("execution %d not found with replication event", task.ExecutionID)
-	}
 
-	rpPolicy, err := replication.PolicyCtl.Get(execution.PolicyID)
+	rpPolicy, err := rep.PolicyCtl.Get(execution.PolicyID)
 	if err != nil {
 		log.Errorf("failed to get replication policy %d: error: %v", execution.PolicyID, err)
 		return nil, nil, err
@@ -107,7 +103,7 @@ func constructReplicationPayload(event *event.ReplicationEvent) (*model.Payload,
 		remoteRegID = rpPolicy.DestRegistry.ID
 	}
 
-	remoteRegistry, err := replication.RegistryMgr.Get(remoteRegID)
+	remoteRegistry, err := rep.RegistryMgr.Get(remoteRegID)
 	if err != nil {
 		log.Errorf("failed to get replication remoteRegistry registry %d: error: %v", remoteRegID, err)
 		return nil, nil, err
@@ -116,8 +112,8 @@ func constructReplicationPayload(event *event.ReplicationEvent) (*model.Payload,
 		return nil, nil, fmt.Errorf("registry %d not found with replication event", remoteRegID)
 	}
 
-	srcNamespace, srcNameAndTag := getMetadataFromResource(task.SrcResource)
+	srcNamespace, srcNameAndTag := getMetadataFromResource(task.SourceResource)
-	destNamespace, destNameAndTag := getMetadataFromResource(task.DstResource)
+	destNamespace, destNameAndTag := getMetadataFromResource(task.DestinationResource)
 
 	extURL, err := config.ExtURL()
 	if err != nil {

src/controller/event/handler/webhook/artifact/replication_test.go

@@ -22,13 +22,14 @@ import (
 	"github.com/goharbor/harbor/src/common/models"
 	"github.com/goharbor/harbor/src/controller/event"
 	"github.com/goharbor/harbor/src/controller/project"
+	rep "github.com/goharbor/harbor/src/controller/replication"
 	"github.com/goharbor/harbor/src/core/config"
 	"github.com/goharbor/harbor/src/lib/q"
 	"github.com/goharbor/harbor/src/pkg/notification"
 	"github.com/goharbor/harbor/src/replication"
-	daoModels "github.com/goharbor/harbor/src/replication/dao/models"
 	"github.com/goharbor/harbor/src/replication/model"
 	projecttesting "github.com/goharbor/harbor/src/testing/controller/project"
+	replicationtesting "github.com/goharbor/harbor/src/testing/controller/replication"
 	"github.com/goharbor/harbor/src/testing/mock"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -40,9 +41,6 @@ type fakedNotificationPolicyMgr struct {
 type fakedReplicationPolicyMgr struct {
 }
 
-type fakedReplicationMgr struct {
-}
-
 type fakedReplicationRegistryMgr struct {
 }
 
@@ -89,46 +87,6 @@ func (f *fakedNotificationPolicyMgr) GetRelatedPolices(int64, string) ([]*models
 	}, nil
 }
 
-func (f *fakedReplicationMgr) StartReplication(policy *model.Policy, resource *model.Resource, trigger model.TriggerType) (int64, error) {
-	return 0, nil
-}
-func (f *fakedReplicationMgr) StopReplication(int64) error {
-	return nil
-}
-func (f *fakedReplicationMgr) ListExecutions(...*daoModels.ExecutionQuery) (int64, []*daoModels.Execution, error) {
-	return 0, nil, nil
-}
-func (f *fakedReplicationMgr) GetExecution(int64) (*daoModels.Execution, error) {
-	return &daoModels.Execution{
-		PolicyID: 1,
-		Trigger:  "manual",
-	}, nil
-}
-func (f *fakedReplicationMgr) ListTasks(...*daoModels.TaskQuery) (int64, []*daoModels.Task, error) {
-	return 0, nil, nil
-}
-func (f *fakedReplicationMgr) GetTask(id int64) (*daoModels.Task, error) {
-	if id == 1 {
-		return &daoModels.Task{
-			ExecutionID: 1,
-			// project info not included when replicating with docker registry
-			SrcResource: "alpine:[v1]",
-			DstResource: "gxt/alpine:[v1] ",
-		}, nil
-	}
-	return &daoModels.Task{
-		ExecutionID: 1,
-		SrcResource: "library/alpine:[v1]",
-		DstResource: "gxt/alpine:[v1] ",
-	}, nil
-}
-func (f *fakedReplicationMgr) UpdateTaskStatus(id int64, status string, statusRevision int64, statusCondition ...string) error {
-	return nil
-}
-func (f *fakedReplicationMgr) GetTaskLog(int64) ([]byte, error) {
-	return nil, nil
-}
-
 // Create new policy
 func (f *fakedReplicationPolicyMgr) Create(*model.Policy) (int64, error) {
 	return 0, nil
@@ -213,24 +171,27 @@ func TestReplicationHandler_Handle(t *testing.T) {
 	config.Init()
 
 	PolicyMgr := notification.PolicyMgr
-	execution := replication.OperationCtl
 	rpPolicy := replication.PolicyCtl
 	rpRegistry := replication.RegistryMgr
 	prj := project.Ctl
+	repCtl := rep.Ctl
 
 	defer func() {
 		notification.PolicyMgr = PolicyMgr
-		replication.OperationCtl = execution
 		replication.PolicyCtl = rpPolicy
 		replication.RegistryMgr = rpRegistry
 		project.Ctl = prj
+		rep.Ctl = repCtl
 	}()
 	notification.PolicyMgr = &fakedNotificationPolicyMgr{}
-	replication.OperationCtl = &fakedReplicationMgr{}
 	replication.PolicyCtl = &fakedReplicationPolicyMgr{}
 	replication.RegistryMgr = &fakedReplicationRegistryMgr{}
 	projectCtl := &projecttesting.Controller{}
 	project.Ctl = projectCtl
+	mockRepCtl := &replicationtesting.Controller{}
+	rep.Ctl = mockRepCtl
+	mockRepCtl.On("GetTask", mock.Anything, mock.Anything).Return(&rep.Task{}, nil)
+	mockRepCtl.On("GetExecution", mock.Anything, mock.Anything).Return(&rep.Execution{}, nil)
 
 	mock.OnAnything(projectCtl, "GetByName").Return(&models.Project{ProjectID: 1}, nil)
 

src/controller/replication/controller.go

@@ -0,0 +1,242 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package replication
+
+import (
+	"context"
+	"time"
+
+	"github.com/goharbor/harbor/src/controller/replication/flow"
+	"github.com/goharbor/harbor/src/jobservice/job"
+	"github.com/goharbor/harbor/src/lib"
+	"github.com/goharbor/harbor/src/lib/errors"
+	"github.com/goharbor/harbor/src/lib/log"
+	"github.com/goharbor/harbor/src/lib/orm"
+	"github.com/goharbor/harbor/src/lib/q"
+	"github.com/goharbor/harbor/src/pkg/task"
+	"github.com/goharbor/harbor/src/replication/model"
+)
+
+// Controller defines the operations related with replication
+type Controller interface {
+	// Start the replication according to the policy
+	Start(ctx context.Context, policy *model.Policy, resource *model.Resource, trigger string) (executionID int64, err error)
+	// Stop the replication specified by the execution ID
+	Stop(ctx context.Context, executionID int64) (err error)
+	// ExecutionCount returns the total count of executions according to the query
+	ExecutionCount(ctx context.Context, query *q.Query) (count int64, err error)
+	// ListExecutions lists the executions according to the query
+	ListExecutions(ctx context.Context, query *q.Query) (executions []*Execution, err error)
+	// GetExecution gets the specific execution
+	GetExecution(ctx context.Context, executionID int64) (execution *Execution, err error)
+	// TaskCount returns the total count of tasks according to the query
+	TaskCount(ctx context.Context, query *q.Query) (count int64, err error)
+	// ListTasks lists the tasks according to the query
+	ListTasks(ctx context.Context, query *q.Query) (tasks []*Task, err error)
+	// GetTask gets the specific task
+	GetTask(ctx context.Context, taskID int64) (task *Task, err error)
+	// GetTaskLog gets the log of the specific task
+	GetTaskLog(ctx context.Context, taskID int64) (log []byte, err error)
+}
+
+var (
+	// Ctl is a global replication controller instance
+	Ctl            = NewController()
+	_   Controller = &controller{}
+)
+
+// NewController creates a new instance of the replication controller
+func NewController() Controller {
+	return &controller{
+		execMgr:    task.ExecMgr,
+		taskMgr:    task.Mgr,
+		flowCtl:    flow.NewController(),
+		ormCreator: orm.Crt,
+		wp:         lib.NewWorkerPool(1024),
+	}
+}
+
+type controller struct {
+	execMgr    task.ExecutionManager
+	taskMgr    task.Manager
+	flowCtl    flow.Controller
+	ormCreator orm.Creator
+	wp         *lib.WorkerPool
+}
+
+func (c *controller) Start(ctx context.Context, policy *model.Policy, resource *model.Resource, trigger string) (int64, error) {
+	logger := log.GetLogger(ctx)
+	if !policy.Enabled {
+		return 0, errors.New(nil).WithCode(errors.PreconditionCode).
+			WithMessage("the policy %d is disabled", policy.ID)
+	}
+	// create an execution record
+	id, err := c.execMgr.Create(ctx, job.Replication, policy.ID, trigger)
+	if err != nil {
+		return 0, err
+	}
+	c.wp.GetWorker()
+	// start the replication flow in background
+	go func() {
+		defer c.wp.ReleaseWorker()
+		// as the process runs inside a goroutine, the transaction in the outer ctx
+		// may be submitted already when the process starts, so create a new context
+		// with orm populated
+		ctxx := orm.NewContext(context.Background(), c.ormCreator.Create())
+		err := c.flowCtl.Start(ctxx, id, policy, resource)
+		if err == nil {
+			// no err, return directly
+			return
+		}
+		// got error, try to stop the execution first in case that some tasks are already created
+		if err := c.execMgr.StopAndWait(ctxx, id, 10*time.Second); err != nil {
+			logger.Errorf("failed to stop the execution %d: %v", id, err)
+		}
+		if err := c.execMgr.MarkError(ctxx, id, err.Error()); err != nil {
+			logger.Errorf("failed to mark error for the execution %d: %v", id, err)
+		}
+	}()
+	return id, nil
+}
+
+func (c *controller) Stop(ctx context.Context, id int64) error {
+	return c.execMgr.Stop(ctx, id)
+}
+
+func (c *controller) ExecutionCount(ctx context.Context, query *q.Query) (int64, error) {
+	query = q.MustClone(query)
+	query.Keywords["VendorType"] = job.Replication
+	return c.execMgr.Count(ctx, query)
+}
+
+func (c *controller) ListExecutions(ctx context.Context, query *q.Query) ([]*Execution, error) {
+	// as the following logic may change the content of the query, clone it first
+	query = q.MustClone(query)
+	query.Keywords["VendorType"] = job.Replication
+	// convert the query keyword "PolicyID" or "policy_id" to the "VendorID"
+	if value, exist := query.Keywords["PolicyID"]; exist {
+		query.Keywords["VendorID"] = value
+		delete(query.Keywords, "PolicyID")
+	}
+	if value, exist := query.Keywords["policy_id"]; exist {
+		query.Keywords["VendorID"] = value
+		delete(query.Keywords, "policy_id")
+	}
+
+	execs, err := c.execMgr.List(ctx, query)
+	if err != nil {
+		return nil, err
+	}
+	var executions []*Execution
+	for _, exec := range execs {
+		executions = append(executions, convertExecution(exec))
+	}
+	return executions, nil
+}
+
+func (c *controller) GetExecution(ctx context.Context, id int64) (*Execution, error) {
+	execs, err := c.execMgr.List(ctx, &q.Query{
+		Keywords: map[string]interface{}{
+			"ID":         id,
+			"VendorType": job.Replication,
+		},
+	})
+	if err != nil {
+		return nil, err
+	}
+	if len(execs) == 0 {
+		return nil, errors.New(nil).WithCode(errors.NotFoundCode).
+			WithMessage("replication execution %d not found", id)
+	}
+	return convertExecution(execs[0]), nil
+}
+
+func (c *controller) TaskCount(ctx context.Context, query *q.Query) (int64, error) {
+	query = q.MustClone(query)
+	query.Keywords["VendorType"] = job.Replication
+	return c.taskMgr.Count(ctx, query)
+}
+
+func (c *controller) ListTasks(ctx context.Context, query *q.Query) ([]*Task, error) {
+	query = q.MustClone(query)
+	query.Keywords["VendorType"] = job.Replication
+	tks, err := c.taskMgr.List(ctx, query)
+	if err != nil {
+		return nil, err
+	}
+	var tasks []*Task
+	for _, tk := range tks {
+		tasks = append(tasks, convertTask(tk))
+	}
+	return tasks, nil
+}
+
+func (c *controller) GetTask(ctx context.Context, id int64) (*Task, error) {
+	tasks, err := c.taskMgr.List(ctx, &q.Query{
+		Keywords: map[string]interface{}{
+			"ID":         id,
+			"VendorType": job.Replication,
+		},
+	})
+	if err != nil {
+		return nil, err
+	}
+	if len(tasks) == 0 {
+		return nil, errors.New(nil).WithCode(errors.NotFoundCode).
+			WithMessage("replication task %d not found", id)
+	}
+	return convertTask(tasks[0]), nil
+}
+
+func (c *controller) GetTaskLog(ctx context.Context, id int64) ([]byte, error) {
+	// make sure the task specified by ID is replication task
+	_, err := c.GetTask(ctx, id)
+	if err != nil {
+		return nil, err
+	}
+	return c.taskMgr.GetLog(ctx, id)
+}
+
+func convertExecution(exec *task.Execution) *Execution {
+	return &Execution{
+		ID:            exec.ID,
+		PolicyID:      exec.VendorID,
+		Status:        exec.Status,
+		StatusMessage: exec.StatusMessage,
+		Metrics:       exec.Metrics,
+		Trigger:       exec.Trigger,
+		StartTime:     exec.StartTime,
+		EndTime:       exec.EndTime,
+	}
+}
+
+func convertTask(task *task.Task) *Task {
+	return &Task{
+		ID:                  task.ID,
+		ExecutionID:         task.ExecutionID,
+		Status:              task.Status,
+		StatusMessage:       task.StatusMessage,
+		RunCount:            task.RunCount,
+		ResourceType:        task.GetStringFromExtraAttrs("resource_type"),
+		SourceResource:      task.GetStringFromExtraAttrs("source_resource"),
+		DestinationResource: task.GetStringFromExtraAttrs("destination_resource"),
+		Operation:           task.GetStringFromExtraAttrs("operation"),
+		JobID:               task.JobID,
+		CreationTime:        task.CreationTime,
+		StartTime:           task.StartTime,
+		UpdateTime:          task.UpdateTime,
+		EndTime:             task.EndTime,
+	}
+}

src/controller/replication/controller_test.go

@@ -0,0 +1,223 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package replication
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/goharbor/harbor/src/jobservice/job"
+	"github.com/goharbor/harbor/src/lib"
+	"github.com/goharbor/harbor/src/pkg/task"
+	"github.com/goharbor/harbor/src/pkg/task/dao"
+	"github.com/goharbor/harbor/src/replication/model"
+	"github.com/goharbor/harbor/src/testing/lib/orm"
+	"github.com/goharbor/harbor/src/testing/mock"
+	testingTask "github.com/goharbor/harbor/src/testing/pkg/task"
+	"github.com/stretchr/testify/suite"
+)
+
+type replicationTestSuite struct {
+	suite.Suite
+	ctl        *controller
+	execMgr    *testingTask.ExecutionManager
+	taskMgr    *testingTask.Manager
+	flowCtl    *flowController
+	ormCreator *orm.Creator
+}
+
+func (r *replicationTestSuite) SetupSuite() {
+	r.execMgr = &testingTask.ExecutionManager{}
+	r.taskMgr = &testingTask.Manager{}
+	r.flowCtl = &flowController{}
+	r.ormCreator = &orm.Creator{}
+	r.ctl = &controller{
+		execMgr:    r.execMgr,
+		taskMgr:    r.taskMgr,
+		flowCtl:    r.flowCtl,
+		ormCreator: r.ormCreator,
+		wp:         lib.NewWorkerPool(1024),
+	}
+}
+
+func (r *replicationTestSuite) TestStart() {
+	// policy is disabled
+	id, err := r.ctl.Start(context.Background(), &model.Policy{Enabled: false}, nil, task.ExecutionTriggerManual)
+	r.Require().NotNil(err)
+
+	// got error when running the replication flow
+	r.execMgr.On("Create", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil)
+	r.execMgr.On("StopAndWait", mock.Anything, mock.Anything, mock.Anything).Return(nil)
+	r.execMgr.On("MarkError", mock.Anything, mock.Anything, mock.Anything).Return(nil)
+	r.flowCtl.On("Start", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(fmt.Errorf("error"))
+	r.ormCreator.On("Create").Return(nil)
+	id, err = r.ctl.Start(context.Background(), &model.Policy{Enabled: true}, nil, task.ExecutionTriggerManual)
+	r.Require().Nil(err)
+	r.Equal(int64(1), id)
+	time.Sleep(1 * time.Second) // wait the functions called in the goroutine
+	r.execMgr.AssertExpectations(r.T())
+	r.flowCtl.AssertExpectations(r.T())
+	r.ormCreator.AssertExpectations(r.T())
+
+	// reset the mocks
+	r.SetupSuite()
+
+	// got no error when running the replication flow
+	r.execMgr.On("Create", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil)
+	r.flowCtl.On("Start", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil)
+	r.ormCreator.On("Create").Return(nil)
+	id, err = r.ctl.Start(context.Background(), &model.Policy{Enabled: true}, nil, task.ExecutionTriggerManual)
+	r.Require().Nil(err)
+	r.Equal(int64(1), id)
+	time.Sleep(1 * time.Second) // wait the functions called in the goroutine
+	r.execMgr.AssertExpectations(r.T())
+	r.flowCtl.AssertExpectations(r.T())
+	r.ormCreator.AssertExpectations(r.T())
+}
+
+func (r *replicationTestSuite) TestStop() {
+	r.execMgr.On("Stop", mock.Anything, mock.Anything).Return(nil)
+	err := r.ctl.Stop(nil, 1)
+	r.Require().Nil(err)
+	r.execMgr.AssertExpectations(r.T())
+}
+
+func (r *replicationTestSuite) TestExecutionCount() {
+	r.execMgr.On("Count", mock.Anything, mock.Anything).Return(int64(1), nil)
+	total, err := r.ctl.ExecutionCount(nil, nil)
+	r.Require().Nil(err)
+	r.Equal(int64(1), total)
+	r.execMgr.AssertExpectations(r.T())
+}
+
+func (r *replicationTestSuite) TestListExecutions() {
+	r.execMgr.On("List", mock.Anything, mock.Anything).Return([]*task.Execution{
+		{
+			ID:         1,
+			VendorType: job.Replication,
+			VendorID:   1,
+			Status:     job.RunningStatus.String(),
+			Metrics: &dao.Metrics{
+				TaskCount:        1,
+				RunningTaskCount: 1,
+			},
+			Trigger:   task.ExecutionTriggerManual,
+			StartTime: time.Time{},
+			EndTime:   time.Time{},
+		},
+	}, nil)
+	executions, err := r.ctl.ListExecutions(nil, nil)
+	r.Require().Nil(err)
+	r.Require().Len(executions, 1)
+	r.Equal(int64(1), executions[0].ID)
+	r.Equal(int64(1), executions[0].PolicyID)
+	r.execMgr.AssertExpectations(r.T())
+}
+
+func (r *replicationTestSuite) TestGetExecution() {
+	r.execMgr.On("List", mock.Anything, mock.Anything).Return([]*task.Execution{
+		{
+			ID:         1,
+			VendorType: job.Replication,
+			VendorID:   1,
+			Status:     job.RunningStatus.String(),
+			Metrics: &dao.Metrics{
+				TaskCount:        1,
+				RunningTaskCount: 1,
+			},
+			Trigger:   task.ExecutionTriggerManual,
+			StartTime: time.Time{},
+			EndTime:   time.Time{},
+		},
+	}, nil)
+	execution, err := r.ctl.GetExecution(nil, 1)
+	r.Require().Nil(err)
+	r.Equal(int64(1), execution.ID)
+	r.Equal(int64(1), execution.PolicyID)
+	r.execMgr.AssertExpectations(r.T())
+}
+
+func (r *replicationTestSuite) TestTaskCount() {
+	r.taskMgr.On("Count", mock.Anything, mock.Anything).Return(int64(1), nil)
+	total, err := r.ctl.TaskCount(nil, nil)
+	r.Require().Nil(err)
+	r.Equal(int64(1), total)
+	r.taskMgr.AssertExpectations(r.T())
+}
+
+func (r *replicationTestSuite) TestListTasks() {
+	r.taskMgr.On("List", mock.Anything, mock.Anything).Return([]*task.Task{
+		{
+			ID:          1,
+			ExecutionID: 1,
+			Status:      job.RunningStatus.String(),
+			ExtraAttrs: map[string]interface{}{
+				"resource_type":        "artifact",
+				"source_resource":      "library/hello-world",
+				"destination_resource": "library/hello-world",
+				"operation":            "copy",
+			},
+		},
+	}, nil)
+	tasks, err := r.ctl.ListTasks(nil, nil)
+	r.Require().Nil(err)
+	r.Require().Len(tasks, 1)
+	r.Equal(int64(1), tasks[0].ID)
+	r.Equal(int64(1), tasks[0].ExecutionID)
+	r.Equal("artifact", tasks[0].ResourceType)
+	r.Equal("library/hello-world", tasks[0].SourceResource)
+	r.Equal("library/hello-world", tasks[0].DestinationResource)
+	r.Equal("copy", tasks[0].Operation)
+	r.taskMgr.AssertExpectations(r.T())
+}
+
+func (r *replicationTestSuite) TestGetTask() {
+	r.taskMgr.On("List", mock.Anything, mock.Anything).Return([]*task.Task{
+		{
+			ID:          1,
+			ExecutionID: 1,
+			Status:      job.RunningStatus.String(),
+			ExtraAttrs: map[string]interface{}{
+				"resource_type":        "artifact",
+				"source_resource":      "library/hello-world",
+				"destination_resource": "library/hello-world",
+				"operation":            "copy",
+			},
+		},
+	}, nil)
+	task, err := r.ctl.GetTask(nil, 1)
+	r.Require().Nil(err)
+	r.Equal(int64(1), task.ID)
+	r.Equal(int64(1), task.ExecutionID)
+	r.Equal("artifact", task.ResourceType)
+	r.Equal("library/hello-world", task.SourceResource)
+	r.Equal("library/hello-world", task.DestinationResource)
+	r.Equal("copy", task.Operation)
+	r.taskMgr.AssertExpectations(r.T())
+}
+
+func (r *replicationTestSuite) TestGetTaskLog() {
+	r.taskMgr.On("GetLog", mock.Anything, mock.Anything).Return([]byte{'a'}, nil)
+	data, err := r.ctl.GetTaskLog(nil, 1)
+	r.Require().Nil(err)
+	r.Equal([]byte{'a'}, data)
+	r.taskMgr.AssertExpectations(r.T())
+}
+
+func TestReplicationTestSuite(t *testing.T) {
+	suite.Run(t, &replicationTestSuite{})
+}

src/controller/replication/flow/controller.go

@@ -0,0 +1,51 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package flow
+
+import (
+	"context"
+
+	"github.com/goharbor/harbor/src/replication/model"
+)
+
+// Flow defines a specific replication flow
+type Flow interface {
+	Run(ctx context.Context) (err error)
+}
+
+// Controller controls the replication flow
+type Controller interface {
+	Start(ctx context.Context, executionID int64, policy *model.Policy, resource *model.Resource) (err error)
+}
+
+// NewController returns an instance of the default flow controller
+func NewController() Controller {
+	return &controller{}
+}
+
+type controller struct{}
+
+func (c *controller) Start(ctx context.Context, executionID int64, policy *model.Policy, resource *model.Resource) error {
+	// deletion flow
+	if resource != nil && resource.Deleted {
+		return NewDeletionFlow(executionID, policy, resource).Run(ctx)
+	}
+	// copy flow
+	resources := []*model.Resource{}
+	if resource != nil {
+		resources = append(resources, resource)
+	}
+	return NewCopyFlow(executionID, policy, resources...).Run(ctx)
+}

src/controller/replication/flow/copy.go

@@ -0,0 +1,130 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package flow
+
+import (
+	"context"
+	"encoding/json"
+
+	"github.com/goharbor/harbor/src/jobservice/job"
+	"github.com/goharbor/harbor/src/lib/log"
+	"github.com/goharbor/harbor/src/pkg/task"
+	"github.com/goharbor/harbor/src/replication/model"
+)
+
+type copyFlow struct {
+	executionID  int64
+	resources    []*model.Resource
+	policy       *model.Policy
+	executionMgr task.ExecutionManager
+	taskMgr      task.Manager
+}
+
+// NewCopyFlow returns an instance of the copy flow which replicates the resources from
+// the source registry to the destination registry. If the parameter "resources" isn't provided,
+// will fetch the resources first
+func NewCopyFlow(executionID int64, policy *model.Policy, resources ...*model.Resource) Flow {
+	return &copyFlow{
+		executionMgr: task.ExecMgr,
+		taskMgr:      task.Mgr,
+		executionID:  executionID,
+		policy:       policy,
+		resources:    resources,
+	}
+}
+
+func (c *copyFlow) Run(ctx context.Context) error {
+	logger := log.GetLogger(ctx)
+	srcAdapter, dstAdapter, err := initialize(c.policy)
+	if err != nil {
+		return err
+	}
+	var srcResources []*model.Resource
+	if len(c.resources) > 0 {
+		srcResources, err = filterResources(c.resources, c.policy.Filters)
+	} else {
+		srcResources, err = fetchResources(srcAdapter, c.policy)
+	}
+	if err != nil {
+		return err
+	}
+
+	isStopped, err := c.isExecutionStopped(ctx)
+	if err != nil {
+		return err
+	}
+	if isStopped {
+		logger.Debugf("the execution %d is stopped, stop the flow", c.executionID)
+		return nil
+	}
+
+	if len(srcResources) == 0 {
+		// no candidates, mark the execution as done directly
+		if err := c.executionMgr.MarkDone(ctx, c.executionID, "no resources need to be replicated"); err != nil {
+			logger.Errorf("failed to mark done for the execution %d: %v", c.executionID, err)
+		}
+		return nil
+	}
+
+	srcResources = assembleSourceResources(srcResources, c.policy)
+	dstResources := assembleDestinationResources(srcResources, c.policy)
+
+	if err = prepareForPush(dstAdapter, dstResources); err != nil {
+		return err
+	}
+
+	return c.createTasks(ctx, srcResources, dstResources)
+}
+
+func (c *copyFlow) isExecutionStopped(ctx context.Context) (bool, error) {
+	execution, err := c.executionMgr.Get(ctx, c.executionID)
+	if err != nil {
+		return false, err
+	}
+	return execution.Status == job.StoppedStatus.String(), nil
+}
+
+func (c *copyFlow) createTasks(ctx context.Context, srcResources, dstResources []*model.Resource) error {
+	for i, resource := range srcResources {
+		src, err := json.Marshal(resource)
+		if err != nil {
+			return err
+		}
+		dest, err := json.Marshal(dstResources[i])
+		if err != nil {
+			return err
+		}
+
+		job := &task.Job{
+			Name: job.Replication,
+			Metadata: &job.Metadata{
+				JobKind: job.KindGeneric,
+			},
+			Parameters: map[string]interface{}{
+				"src_resource": string(src),
+				"dst_resource": string(dest),
+			},
+		}
+
+		if _, err = c.taskMgr.Create(ctx, c.executionID, job, map[string]interface{}{
+			"operation":            "copy",
+			"resource_type":        string(resource.Type),
+			"source_resource":      getResourceName(resource),
+			"destination_resource": getResourceName(dstResources[i])}); err != nil {
+			return err
+		}
+	}
+	return nil
+}

src/controller/replication/flow/copy_test.go

@@ -0,0 +1,83 @@
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package flow
+
+import (
+	"context"
+	"github.com/goharbor/harbor/src/jobservice/job"
+	"github.com/goharbor/harbor/src/replication/adapter"
+	"github.com/stretchr/testify/mock"
+	"testing"
+
+	"github.com/goharbor/harbor/src/pkg/task"
+	"github.com/goharbor/harbor/src/replication/model"
+	testingTask "github.com/goharbor/harbor/src/testing/pkg/task"
+	"github.com/stretchr/testify/suite"
+)
+
+type copyFlowTestSuite struct {
+	suite.Suite
+}
+
+func (c *copyFlowTestSuite) TestRun() {
+	adp := &mockAdapter{}
+	factory := &mockFactory{}
+	factory.On("AdapterPattern").Return(nil)
+	factory.On("Create", mock.Anything).Return(adp, nil)
+	adapter.RegisterFactory("TEST_FOR_COPY_FLOW", factory)
+
+	adp.On("Info").Return(&model.RegistryInfo{
+		SupportedResourceTypes: []model.ResourceType{
+			model.ResourceTypeArtifact,
+		},
+	}, nil)
+	adp.On("FetchArtifacts", mock.Anything).Return([]*model.Resource{
+		{
+			Type: model.ResourceTypeChart,
+			Metadata: &model.ResourceMetadata{
+				Repository: &model.Repository{
+					Name: "library/hello-world",
+				},
+				Vtags: []string{"latest"},
+			},
+			Override: false,
+		},
+	}, nil)
+	adp.On("PrepareForPush", mock.Anything).Return(nil)
+
+	execMgr := &testingTask.ExecutionManager{}
+	execMgr.On("Get", mock.Anything, mock.Anything).Return(&task.Execution{
+		Status: job.RunningStatus.String(),
+	}, nil)
+
+	taskMgr := &testingTask.Manager{}
+	taskMgr.On("Create", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil)
+	policy := &model.Policy{
+		SrcRegistry: &model.Registry{
+			Type: "TEST_FOR_COPY_FLOW",
+		},
+		DestRegistry: &model.Registry{
+			Type: "TEST_FOR_COPY_FLOW",
+		},
+	}
+	flow := &copyFlow{
+		executionID:  1,
+		policy:       policy,
+		executionMgr: execMgr,
+		taskMgr:      taskMgr,
+	}
+	err := flow.Run(context.Background())
+	c.Require().Nil(err)
+}
+
+func TestCopyFlowTestSuite(t *testing.T) {
+	suite.Run(t, &copyFlowTestSuite{})
+}

src/controller/replication/flow/deletion.go

@@ -0,0 +1,103 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package flow
+
+import (
+	"context"
+	"encoding/json"
+
+	"github.com/goharbor/harbor/src/jobservice/job"
+	"github.com/goharbor/harbor/src/lib/log"
+	"github.com/goharbor/harbor/src/pkg/task"
+	"github.com/goharbor/harbor/src/replication/model"
+)
+
+type deletionFlow struct {
+	executionID  int64
+	policy       *model.Policy
+	executionMgr task.ExecutionManager
+	taskMgr      task.Manager
+	resources    []*model.Resource
+}
+
+// NewDeletionFlow returns an instance of the delete flow which deletes the resources
+// on the destination registry
+func NewDeletionFlow(executionID int64, policy *model.Policy, resources ...*model.Resource) Flow {
+	return &deletionFlow{
+		executionMgr: task.ExecMgr,
+		taskMgr:      task.Mgr,
+		executionID:  executionID,
+		policy:       policy,
+		resources:    resources,
+	}
+}
+
+func (d *deletionFlow) Run(ctx context.Context) error {
+	logger := log.GetLogger(ctx)
+	srcResources, err := filterResources(d.resources, d.policy.Filters)
+	if err != nil {
+		return err
+	}
+	if len(srcResources) == 0 {
+		// no candidates, mark the execution as done directly
+		if err := d.executionMgr.MarkDone(ctx, d.executionID, "no resources need to be replicated"); err != nil {
+			logger.Errorf("failed to mark done for the execution %d: %v", d.executionID, err)
+		}
+		return nil
+	}
+
+	srcResources = assembleSourceResources(srcResources, d.policy)
+	dstResources := assembleDestinationResources(srcResources, d.policy)
+
+	return d.createTasks(ctx, srcResources, dstResources)
+}
+
+func (d *deletionFlow) createTasks(ctx context.Context, srcResources, dstResources []*model.Resource) error {
+	for i, resource := range srcResources {
+		src, err := json.Marshal(resource)
+		if err != nil {
+			return err
+		}
+		dest, err := json.Marshal(dstResources[i])
+		if err != nil {
+			return err
+		}
+
+		job := &task.Job{
+			Name: job.Replication,
+			Metadata: &job.Metadata{
+				JobKind: job.KindGeneric,
+			},
+			Parameters: map[string]interface{}{
+				"src_resource": string(src),
+				"dst_resource": string(dest),
+			},
+		}
+
+		operation := "deletion"
+		if dstResources[i].IsDeleteTag {
+			operation = "tag deletion"
+		}
+
+		if _, err = d.taskMgr.Create(ctx, d.executionID, job, map[string]interface{}{
+			"operation":            operation,
+			"resource_type":        string(resource.Type),
+			"source_resource":      getResourceName(resource),
+			"destination_resource": getResourceName(dstResources[i])}); err != nil {
+			return err
+		}
+	}
+	return nil
+}

src/controller/replication/flow/deletion_test.go

@@ -15,16 +15,23 @@
 package flow
 
 import (
+	"context"
 	"testing"
 
 	"github.com/goharbor/harbor/src/replication/model"
-	"github.com/stretchr/testify/assert"
+	"github.com/goharbor/harbor/src/testing/pkg/task"
-	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/suite"
 )
 
-func TestRunOfDeletionFlow(t *testing.T) {
+type deletionFlowTestSuite struct {
-	scheduler := &fakedScheduler{}
+	suite.Suite
-	executionMgr := &fakedExecutionManager{}
+}
+
+func (d *deletionFlowTestSuite) TestRun() {
+	taskMgr := &task.Manager{}
+	taskMgr.On("Create", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil)
+
 	policy := &model.Policy{
 		SrcRegistry: &model.Registry{
 			Type: model.RegistryTypeHarbor,
@@ -47,8 +54,16 @@ func TestRunOfDeletionFlow(t *testing.T) {
 			},
 		},
 	}
-	flow := NewDeletionFlow(executionMgr, scheduler, 1, policy, resources...)
+	flow := &deletionFlow{
-	n, err := flow.Run(nil)
+		executionID: 1,
-	require.Nil(t, err)
+		policy:      policy,
-	assert.Equal(t, 1, n)
+		taskMgr:     taskMgr,
+		resources:   resources,
+	}
+	err := flow.Run(context.Background())
+	d.Require().Nil(err)
+}
+
+func TestDeletionFlowTestSuite(t *testing.T) {
+	suite.Run(t, &deletionFlowTestSuite{})
 }

src/controller/replication/flow/mock.go

@@ -0,0 +1,28 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package flow
+
+import (
+	"github.com/goharbor/harbor/src/replication/adapter"
+)
+
+// define a new interface to combine the two interfaces of adapter for mockery to generate the mocks
+type registryAdapter interface {
+	adapter.Adapter
+	adapter.ArtifactRegistry
+}
+
+//go:generate mockery --dir . --name registryAdapter --output . --outpkg flow --filename mock_adapter_test.go --structname mockAdapter
+//go:generate mockery --dir ../../../replication/adapter --name Factory --output . --outpkg flow --filename mock_adapter_factory_test.go --structname mockFactory

src/controller/replication/flow/mock_adapter_factory_test.go

@@ -0,0 +1,54 @@
+// Code generated by mockery v2.1.0. DO NOT EDIT.
+
+package flow
+
+import (
+	adapter "github.com/goharbor/harbor/src/replication/adapter"
+	mock "github.com/stretchr/testify/mock"
+
+	model "github.com/goharbor/harbor/src/replication/model"
+)
+
+// mockFactory is an autogenerated mock type for the Factory type
+type mockFactory struct {
+	mock.Mock
+}
+
+// AdapterPattern provides a mock function with given fields:
+func (_m *mockFactory) AdapterPattern() *model.AdapterPattern {
+	ret := _m.Called()
+
+	var r0 *model.AdapterPattern
+	if rf, ok := ret.Get(0).(func() *model.AdapterPattern); ok {
+		r0 = rf()
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.AdapterPattern)
+		}
+	}
+
+	return r0
+}
+
+// Create provides a mock function with given fields: _a0
+func (_m *mockFactory) Create(_a0 *model.Registry) (adapter.Adapter, error) {
+	ret := _m.Called(_a0)
+
+	var r0 adapter.Adapter
+	if rf, ok := ret.Get(0).(func(*model.Registry) adapter.Adapter); ok {
+		r0 = rf(_a0)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(adapter.Adapter)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(*model.Registry) error); ok {
+		r1 = rf(_a0)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}

src/controller/replication/flow/mock_adapter_test.go

@@ -0,0 +1,278 @@
+// Code generated by mockery v2.1.0. DO NOT EDIT.
+
+package flow
+
+import (
+	distribution "github.com/docker/distribution"
+
+	io "io"
+
+	mock "github.com/stretchr/testify/mock"
+
+	model "github.com/goharbor/harbor/src/replication/model"
+)
+
+// mockAdapter is an autogenerated mock type for the registryAdapter type
+type mockAdapter struct {
+	mock.Mock
+}
+
+// BlobExist provides a mock function with given fields: repository, digest
+func (_m *mockAdapter) BlobExist(repository string, digest string) (bool, error) {
+	ret := _m.Called(repository, digest)
+
+	var r0 bool
+	if rf, ok := ret.Get(0).(func(string, string) bool); ok {
+		r0 = rf(repository, digest)
+	} else {
+		r0 = ret.Get(0).(bool)
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(string, string) error); ok {
+		r1 = rf(repository, digest)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// DeleteManifest provides a mock function with given fields: repository, reference
+func (_m *mockAdapter) DeleteManifest(repository string, reference string) error {
+	ret := _m.Called(repository, reference)
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func(string, string) error); ok {
+		r0 = rf(repository, reference)
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+// DeleteTag provides a mock function with given fields: repository, tag
+func (_m *mockAdapter) DeleteTag(repository string, tag string) error {
+	ret := _m.Called(repository, tag)
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func(string, string) error); ok {
+		r0 = rf(repository, tag)
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+// FetchArtifacts provides a mock function with given fields: filters
+func (_m *mockAdapter) FetchArtifacts(filters []*model.Filter) ([]*model.Resource, error) {
+	ret := _m.Called(filters)
+
+	var r0 []*model.Resource
+	if rf, ok := ret.Get(0).(func([]*model.Filter) []*model.Resource); ok {
+		r0 = rf(filters)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).([]*model.Resource)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func([]*model.Filter) error); ok {
+		r1 = rf(filters)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// HealthCheck provides a mock function with given fields:
+func (_m *mockAdapter) HealthCheck() (model.HealthStatus, error) {
+	ret := _m.Called()
+
+	var r0 model.HealthStatus
+	if rf, ok := ret.Get(0).(func() model.HealthStatus); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Get(0).(model.HealthStatus)
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func() error); ok {
+		r1 = rf()
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// Info provides a mock function with given fields:
+func (_m *mockAdapter) Info() (*model.RegistryInfo, error) {
+	ret := _m.Called()
+
+	var r0 *model.RegistryInfo
+	if rf, ok := ret.Get(0).(func() *model.RegistryInfo); ok {
+		r0 = rf()
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.RegistryInfo)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func() error); ok {
+		r1 = rf()
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// ManifestExist provides a mock function with given fields: repository, reference
+func (_m *mockAdapter) ManifestExist(repository string, reference string) (bool, string, error) {
+	ret := _m.Called(repository, reference)
+
+	var r0 bool
+	if rf, ok := ret.Get(0).(func(string, string) bool); ok {
+		r0 = rf(repository, reference)
+	} else {
+		r0 = ret.Get(0).(bool)
+	}
+
+	var r1 string
+	if rf, ok := ret.Get(1).(func(string, string) string); ok {
+		r1 = rf(repository, reference)
+	} else {
+		r1 = ret.Get(1).(string)
+	}
+
+	var r2 error
+	if rf, ok := ret.Get(2).(func(string, string) error); ok {
+		r2 = rf(repository, reference)
+	} else {
+		r2 = ret.Error(2)
+	}
+
+	return r0, r1, r2
+}
+
+// PrepareForPush provides a mock function with given fields: _a0
+func (_m *mockAdapter) PrepareForPush(_a0 []*model.Resource) error {
+	ret := _m.Called(_a0)
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func([]*model.Resource) error); ok {
+		r0 = rf(_a0)
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+// PullBlob provides a mock function with given fields: repository, digest
+func (_m *mockAdapter) PullBlob(repository string, digest string) (int64, io.ReadCloser, error) {
+	ret := _m.Called(repository, digest)
+
+	var r0 int64
+	if rf, ok := ret.Get(0).(func(string, string) int64); ok {
+		r0 = rf(repository, digest)
+	} else {
+		r0 = ret.Get(0).(int64)
+	}
+
+	var r1 io.ReadCloser
+	if rf, ok := ret.Get(1).(func(string, string) io.ReadCloser); ok {
+		r1 = rf(repository, digest)
+	} else {
+		if ret.Get(1) != nil {
+			r1 = ret.Get(1).(io.ReadCloser)
+		}
+	}
+
+	var r2 error
+	if rf, ok := ret.Get(2).(func(string, string) error); ok {
+		r2 = rf(repository, digest)
+	} else {
+		r2 = ret.Error(2)
+	}
+
+	return r0, r1, r2
+}
+
+// PullManifest provides a mock function with given fields: repository, reference, accepttedMediaTypes
+func (_m *mockAdapter) PullManifest(repository string, reference string, accepttedMediaTypes ...string) (distribution.Manifest, string, error) {
+	_va := make([]interface{}, len(accepttedMediaTypes))
+	for _i := range accepttedMediaTypes {
+		_va[_i] = accepttedMediaTypes[_i]
+	}
+	var _ca []interface{}
+	_ca = append(_ca, repository, reference)
+	_ca = append(_ca, _va...)
+	ret := _m.Called(_ca...)
+
+	var r0 distribution.Manifest
+	if rf, ok := ret.Get(0).(func(string, string, ...string) distribution.Manifest); ok {
+		r0 = rf(repository, reference, accepttedMediaTypes...)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(distribution.Manifest)
+		}
+	}
+
+	var r1 string
+	if rf, ok := ret.Get(1).(func(string, string, ...string) string); ok {
+		r1 = rf(repository, reference, accepttedMediaTypes...)
+	} else {
+		r1 = ret.Get(1).(string)
+	}
+
+	var r2 error
+	if rf, ok := ret.Get(2).(func(string, string, ...string) error); ok {
+		r2 = rf(repository, reference, accepttedMediaTypes...)
+	} else {
+		r2 = ret.Error(2)
+	}
+
+	return r0, r1, r2
+}
+
+// PushBlob provides a mock function with given fields: repository, digest, size, blob
+func (_m *mockAdapter) PushBlob(repository string, digest string, size int64, blob io.Reader) error {
+	ret := _m.Called(repository, digest, size, blob)
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func(string, string, int64, io.Reader) error); ok {
+		r0 = rf(repository, digest, size, blob)
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+// PushManifest provides a mock function with given fields: repository, reference, mediaType, payload
+func (_m *mockAdapter) PushManifest(repository string, reference string, mediaType string, payload []byte) (string, error) {
+	ret := _m.Called(repository, reference, mediaType, payload)
+
+	var r0 string
+	if rf, ok := ret.Get(0).(func(string, string, string, []byte) string); ok {
+		r0 = rf(repository, reference, mediaType, payload)
+	} else {
+		r0 = ret.Get(0).(string)
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(string, string, string, []byte) error); ok {
+		r1 = rf(repository, reference, mediaType, payload)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}

src/controller/replication/flow/stage.go

@@ -15,17 +15,12 @@
 package flow
 
 import (
-	"errors"
 	"fmt"
-	"github.com/goharbor/harbor/src/replication/filter"
-	"time"
 
 	"github.com/goharbor/harbor/src/lib/log"
 	adp "github.com/goharbor/harbor/src/replication/adapter"
-	"github.com/goharbor/harbor/src/replication/dao/models"
+	"github.com/goharbor/harbor/src/replication/filter"
 	"github.com/goharbor/harbor/src/replication/model"
-	"github.com/goharbor/harbor/src/replication/operation/execution"
-	"github.com/goharbor/harbor/src/replication/operation/scheduler"
 	"github.com/goharbor/harbor/src/replication/util"
 )
 
@@ -171,108 +166,6 @@ func prepareForPush(adapter adp.Adapter, resources []*model.Resource) error {
 	return nil
 }
 
-// preprocess
-func preprocess(scheduler scheduler.Scheduler, srcResources, dstResources []*model.Resource) ([]*scheduler.ScheduleItem, error) {
-	items, err := scheduler.Preprocess(srcResources, dstResources)
-	if err != nil {
-		return nil, fmt.Errorf("failed to preprocess the resources: %v", err)
-	}
-	log.Debug("preprocess the resources completed")
-	return items, nil
-}
-
-// create task records in database
-func createTasks(mgr execution.Manager, executionID int64, items []*scheduler.ScheduleItem) error {
-	for _, item := range items {
-		operation := "copy"
-		if item.DstResource.Deleted {
-			operation = "deletion"
-			if item.DstResource.IsDeleteTag {
-				operation = "tag deletion"
-			}
-		}
-
-		task := &models.Task{
-			ExecutionID:  executionID,
-			Status:       models.TaskStatusInitialized,
-			ResourceType: string(item.SrcResource.Type),
-			SrcResource:  getResourceName(item.SrcResource),
-			DstResource:  getResourceName(item.DstResource),
-			Operation:    operation,
-		}
-
-		id, err := mgr.CreateTask(task)
-		if err != nil {
-			// if failed to create the task for one of the items,
-			// the whole execution is marked as failure and all
-			// the items will not be submitted
-			return fmt.Errorf("failed to create task records for the execution %d: %v", executionID, err)
-		}
-
-		item.TaskID = id
-		log.Debugf("task record %d for the execution %d created", id, executionID)
-	}
-	return nil
-}
-
-// schedule the replication tasks and update the task's status
-// returns the count of tasks which have been scheduled and the error
-func schedule(scheduler scheduler.Scheduler, executionMgr execution.Manager, items []*scheduler.ScheduleItem) (int, error) {
-	results, err := scheduler.Schedule(items)
-	if err != nil {
-		return 0, fmt.Errorf("failed to schedule the tasks: %v", err)
-	}
-
-	allFailed := true
-	n := len(results)
-	for _, result := range results {
-		// if the task is failed to be submitted, update the status of the
-		// task as failure
-		now := time.Now()
-		if result.Error != nil {
-			log.Errorf("failed to schedule the task %d: %v", result.TaskID, result.Error)
-			if err = executionMgr.UpdateTask(&models.Task{
-				ID:      result.TaskID,
-				Status:  models.TaskStatusFailed,
-				EndTime: now,
-			}, "Status", "EndTime"); err != nil {
-				log.Errorf("failed to update the task status %d: %v", result.TaskID, err)
-			}
-			continue
-		}
-		allFailed = false
-		// if the task is submitted successfully, update the status, job ID and start time
-		if err = executionMgr.UpdateTaskStatus(result.TaskID, models.TaskStatusPending, 0, models.TaskStatusInitialized); err != nil {
-			log.Errorf("failed to update the task status %d: %v", result.TaskID, err)
-		}
-		if err = executionMgr.UpdateTask(&models.Task{
-			ID:        result.TaskID,
-			JobID:     result.JobID,
-			StartTime: now,
-		}, "JobID", "StartTime"); err != nil {
-			log.Errorf("failed to update the task %d: %v", result.TaskID, err)
-		}
-		log.Debugf("the task %d scheduled", result.TaskID)
-	}
-	// if all the tasks are failed, return err
-	if allFailed {
-		return n, errors.New("all tasks are failed")
-	}
-	return n, nil
-}
-
-// check whether the execution is stopped
-func isExecutionStopped(mgr execution.Manager, id int64) (bool, error) {
-	execution, err := mgr.Get(id)
-	if err != nil {
-		return false, err
-	}
-	if execution == nil {
-		return false, fmt.Errorf("execution %d not found", id)
-	}
-	return execution.Status == models.ExecutionStatusStopped, nil
-}
-
 // return the name with format "res_name" or "res_name:[vtag1,vtag2,vtag3]"
 // if the resource has vtags
 func getResourceName(res *model.Resource) string {

src/controller/replication/flow/stage_test.go

@@ -0,0 +1,221 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package flow
+
+import (
+	"testing"
+
+	"github.com/goharbor/harbor/src/replication/adapter"
+	"github.com/goharbor/harbor/src/replication/model"
+	"github.com/goharbor/harbor/src/testing/mock"
+	"github.com/stretchr/testify/suite"
+)
+
+type stageTestSuite struct {
+	suite.Suite
+}
+
+func (s *stageTestSuite) SetupTest() {
+}
+
+func (s *stageTestSuite) TestInitialize() {
+	factory := &mockFactory{}
+	factory.On("AdapterPattern").Return(nil)
+	adapter.RegisterFactory(model.RegistryTypeHarbor, factory)
+
+	policy := &model.Policy{
+		SrcRegistry: &model.Registry{
+			Type: model.RegistryTypeHarbor,
+		},
+		DestRegistry: &model.Registry{
+			Type: model.RegistryTypeHarbor,
+		},
+	}
+	factory.On("Create", mock.Anything).Return(&mockAdapter{}, nil)
+	_, _, err := initialize(policy)
+	s.Nil(err)
+	factory.AssertExpectations(s.T())
+}
+
+func (s *stageTestSuite) TestFetchResources() {
+	adapter := &mockAdapter{}
+	adapter.On("Info").Return(&model.RegistryInfo{
+		SupportedResourceTypes: []model.ResourceType{
+			model.ResourceTypeArtifact,
+		},
+	}, nil)
+	adapter.On("FetchArtifacts", mock.Anything).Return([]*model.Resource{
+		{},
+		{},
+	}, nil)
+	policy := &model.Policy{}
+	resources, err := fetchResources(adapter, policy)
+	s.Require().Nil(err)
+	s.Len(resources, 2)
+	adapter.AssertExpectations(s.T())
+}
+
+func (s *stageTestSuite) TestFilterResources() {
+	resources := []*model.Resource{
+		{
+			Type: model.ResourceTypeImage,
+			Metadata: &model.ResourceMetadata{
+				Repository: &model.Repository{
+					Name: "library/hello-world",
+				},
+				Artifacts: []*model.Artifact{
+					{
+						Tags: []string{"latest"},
+					},
+				},
+			},
+			Deleted:  true,
+			Override: true,
+		},
+		{
+			Type: model.ResourceTypeChart,
+			Metadata: &model.ResourceMetadata{
+				Repository: &model.Repository{
+					Name: "library/harbor",
+				},
+				Artifacts: []*model.Artifact{
+					{
+						Tags: []string{"0.2.0"},
+					},
+					{
+						Tags: []string{"0.3.0"},
+					},
+				},
+			},
+			Deleted:  true,
+			Override: true,
+		},
+		{
+			Type: model.ResourceTypeChart,
+			Metadata: &model.ResourceMetadata{
+				Repository: &model.Repository{
+					Name: "library/mysql",
+				},
+				Artifacts: []*model.Artifact{
+					{
+						Tags: []string{"1.0"},
+					},
+				},
+			},
+			Deleted:  true,
+			Override: true,
+		},
+	}
+	filters := []*model.Filter{
+		{
+			Type:  model.FilterTypeResource,
+			Value: model.ResourceTypeChart,
+		},
+		{
+			Type:  model.FilterTypeName,
+			Value: "library/*",
+		},
+		{
+			Type:  model.FilterTypeName,
+			Value: "library/harbor",
+		},
+		{
+			Type:  model.FilterTypeTag,
+			Value: "0.2.?",
+		},
+	}
+	res, err := filterResources(resources, filters)
+	s.Require().Nil(err)
+	s.Len(res, 1)
+	s.Equal("library/harbor", res[0].Metadata.Repository.Name)
+	s.Equal(1, len(res[0].Metadata.Artifacts))
+	s.Equal("0.2.0", res[0].Metadata.Artifacts[0].Tags[0])
+}
+
+func (s *stageTestSuite) TestAssembleSourceResources() {
+	resources := []*model.Resource{
+		{
+			Type: model.ResourceTypeChart,
+			Metadata: &model.ResourceMetadata{
+				Repository: &model.Repository{
+					Name: "library/hello-world",
+				},
+				Vtags: []string{"latest"},
+			},
+			Override: false,
+		},
+	}
+	policy := &model.Policy{
+		SrcRegistry: &model.Registry{
+			ID: 1,
+		},
+	}
+	res := assembleSourceResources(resources, policy)
+	s.Len(res, 1)
+	s.Equal(int64(1), res[0].Registry.ID)
+}
+
+func (s *stageTestSuite) TestAssembleDestinationResources() {
+	resources := []*model.Resource{
+		{
+			Type: model.ResourceTypeChart,
+			Metadata: &model.ResourceMetadata{
+				Repository: &model.Repository{
+					Name: "library/hello-world",
+				},
+				Vtags: []string{"latest"},
+			},
+			Override: false,
+		},
+	}
+	policy := &model.Policy{
+		DestRegistry:  &model.Registry{},
+		DestNamespace: "test",
+		Override:      true,
+	}
+	res := assembleDestinationResources(resources, policy)
+	s.Len(res, 1)
+	s.Equal(model.ResourceTypeChart, res[0].Type)
+	s.Equal("test/hello-world", res[0].Metadata.Repository.Name)
+	s.Equal(1, len(res[0].Metadata.Vtags))
+	s.Equal("latest", res[0].Metadata.Vtags[0])
+}
+
+func (s *stageTestSuite) TestReplaceNamespace() {
+	// empty namespace
+	repository := "c"
+	namespace := ""
+	result := replaceNamespace(repository, namespace)
+	s.Equal("c", result)
+	// repository contains no "/"
+	repository = "c"
+	namespace = "n"
+	result = replaceNamespace(repository, namespace)
+	s.Equal("n/c", result)
+	// repository contains only one "/"
+	repository = "b/c"
+	namespace = "n"
+	result = replaceNamespace(repository, namespace)
+	s.Equal("n/c", result)
+	// repository contains more than one "/"
+	repository = "a/b/c"
+	namespace = "n"
+	result = replaceNamespace(repository, namespace)
+	s.Equal("n/c", result)
+}
+
+func TestStage(t *testing.T) {
+	suite.Run(t, &stageTestSuite{})
+}

src/controller/replication/mock.go

@@ -12,19 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package task
+package replication
 
-import (
+//go:generate mockery --dir ./flow --name Controller --output . --outpkg replication --filename mock_flow_controller_test.go --structname flowController
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestRegisterCheckInProcessor(t *testing.T) {
-	err := RegisterCheckInProcessor("test", nil)
-	assert.Nil(t, err)
-
-	// already exist
-	err = RegisterCheckInProcessor("test", nil)
-	assert.NotNil(t, err)
-}

src/controller/replication/mock_flow_controller_test.go

@@ -0,0 +1,30 @@
+// Code generated by mockery v2.1.0. DO NOT EDIT.
+
+package replication
+
+import (
+	context "context"
+
+	mock "github.com/stretchr/testify/mock"
+
+	model "github.com/goharbor/harbor/src/replication/model"
+)
+
+// flowController is an autogenerated mock type for the Controller type
+type flowController struct {
+	mock.Mock
+}
+
+// Start provides a mock function with given fields: ctx, executionID, policy, resource
+func (_m *flowController) Start(ctx context.Context, executionID int64, policy *model.Policy, resource *model.Resource) error {
+	ret := _m.Called(ctx, executionID, policy, resource)
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func(context.Context, int64, *model.Policy, *model.Resource) error); ok {
+		r0 = rf(ctx, executionID, policy, resource)
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}

src/controller/replication/model.go

@@ -0,0 +1,51 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package replication
+
+import (
+	"time"
+
+	"github.com/goharbor/harbor/src/pkg/task/dao"
+)
+
+// Execution model for replication
+type Execution struct {
+	ID            int64
+	PolicyID      int64
+	Status        string
+	StatusMessage string
+	Metrics       *dao.Metrics
+	Trigger       string
+	StartTime     time.Time
+	EndTime       time.Time
+}
+
+// Task model for replication
+type Task struct {
+	ID                  int64
+	ExecutionID         int64
+	Status              string
+	StatusMessage       string
+	RunCount            int32
+	ResourceType        string
+	SourceResource      string
+	DestinationResource string
+	Operation           string
+	JobID               string
+	CreationTime        time.Time
+	StartTime           time.Time
+	UpdateTime          time.Time
+	EndTime             time.Time
+}

src/core/api/base.go

@@ -15,6 +15,7 @@
 package api
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"net/http"
@@ -30,7 +31,6 @@ import (
 	"github.com/goharbor/harbor/src/core/config"
 	"github.com/goharbor/harbor/src/lib/errors"
 	"github.com/goharbor/harbor/src/lib/log"
-	"github.com/goharbor/harbor/src/lib/orm"
 	"github.com/goharbor/harbor/src/pkg/project"
 	"github.com/goharbor/harbor/src/pkg/repository"
 	"github.com/goharbor/harbor/src/pkg/retention"
@@ -194,13 +194,9 @@ func Init() error {
 
 	retentionController = retention.NewAPIController(retentionMgr, projectMgr, repository.Mgr, scheduler.Sched, retentionLauncher)
 
-	retentionCallbackFun := func(p interface{}) error {
+	retentionCallbackFun := func(ctx context.Context, p string) error {
-		str, ok := p.(string)
-		if !ok {
-			return fmt.Errorf("the type of param %v isn't string", p)
-		}
 		param := &retention.TriggerParam{}
-		if err := json.Unmarshal([]byte(str), param); err != nil {
+		if err := json.Unmarshal([]byte(p), param); err != nil {
 			return fmt.Errorf("failed to unmarshal the param: %v", err)
 		}
 		_, err := retentionController.TriggerRetentionExec(param.PolicyID, param.Trigger, false)
@@ -211,16 +207,12 @@ func Init() error {
 		return err
 	}
 
-	p2pPreheatCallbackFun := func(p interface{}) error {
+	p2pPreheatCallbackFun := func(ctx context.Context, p string) error {
-		str, ok := p.(string)
-		if !ok {
-			return fmt.Errorf("the type of param %v isn't string", p)
-		}
 		param := &preheat.TriggerParam{}
-		if err := json.Unmarshal([]byte(str), param); err != nil {
+		if err := json.Unmarshal([]byte(p), param); err != nil {
 			return fmt.Errorf("failed to unmarshal the param: %v", err)
 		}
-		_, err := preheat.Enf.EnforcePolicy(orm.Context(), param.PolicyID)
+		_, err := preheat.Enf.EnforcePolicy(ctx, param.PolicyID)
 		return err
 	}
 	err = scheduler.RegisterCallbackFunc(preheat.SchedulerCallback, p2pPreheatCallbackFun)

src/core/api/harborapi_test.go

@@ -137,10 +137,6 @@ func init() {
 	beego.Router("/api/projects/:pid([0-9]+)/robots/:id([0-9]+)", &RobotAPI{}, "get:Get;put:Put;delete:Delete")
 
 	beego.Router("/api/replication/adapters", &ReplicationAdapterAPI{}, "get:List")
-	beego.Router("/api/replication/executions", &ReplicationOperationAPI{}, "get:ListExecutions;post:CreateExecution")
-	beego.Router("/api/replication/executions/:id([0-9]+)", &ReplicationOperationAPI{}, "get:GetExecution;put:StopExecution")
-	beego.Router("/api/replication/executions/:id([0-9]+)/tasks", &ReplicationOperationAPI{}, "get:ListTasks")
-	beego.Router("/api/replication/executions/:id([0-9]+)/tasks/:tid([0-9]+)/log", &ReplicationOperationAPI{}, "get:GetTaskLog")
 
 	beego.Router("/api/replication/policies", &ReplicationPolicyAPI{}, "get:List;post:Create")
 	beego.Router("/api/replication/policies/:id([0-9]+)", &ReplicationPolicyAPI{}, "get:Get;put:Update;delete:Delete")

src/core/api/replication_execution.go

@@ -1,248 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package api
-
-import (
-	"errors"
-	"fmt"
-	"net/http"
-	"strconv"
-
-	common_http "github.com/goharbor/harbor/src/common/http"
-	"github.com/goharbor/harbor/src/replication"
-	"github.com/goharbor/harbor/src/replication/dao/models"
-	"github.com/goharbor/harbor/src/replication/event"
-	"github.com/goharbor/harbor/src/replication/model"
-)
-
-// ReplicationOperationAPI handles the replication operation requests
-type ReplicationOperationAPI struct {
-	BaseController
-	execution *models.Execution
-	task      *models.Task
-}
-
-// Prepare ...
-func (r *ReplicationOperationAPI) Prepare() {
-	r.BaseController.Prepare()
-	// As we delegate the jobservice to trigger the scheduled replication,
-	// we need to allow the jobservice to call the API
-	if !(r.SecurityCtx.IsSysAdmin() || r.SecurityCtx.IsSolutionUser()) {
-		if !r.SecurityCtx.IsAuthenticated() {
-			r.SendUnAuthorizedError(errors.New("UnAuthorized"))
-			return
-		}
-		r.SendForbiddenError(errors.New(r.SecurityCtx.GetUsername()))
-		return
-	}
-
-	// check the existence of execution if execution ID is provided in the request path
-	executionIDStr := r.GetStringFromPath(":id")
-	if len(executionIDStr) > 0 {
-		executionID, err := strconv.ParseInt(executionIDStr, 10, 64)
-		if err != nil || executionID <= 0 {
-			r.SendBadRequestError(fmt.Errorf("invalid execution ID: %s", executionIDStr))
-			return
-		}
-		execution, err := replication.OperationCtl.GetExecution(executionID)
-		if err != nil {
-			r.SendInternalServerError(fmt.Errorf("failed to get execution %d: %v", executionID, err))
-			return
-		}
-		if execution == nil {
-			r.SendNotFoundError(fmt.Errorf("execution %d not found", executionID))
-			return
-		}
-		r.execution = execution
-
-		// check the existence of task if task ID is provided in the request path
-		taskIDStr := r.GetStringFromPath(":tid")
-		if len(taskIDStr) > 0 {
-			taskID, err := strconv.ParseInt(taskIDStr, 10, 64)
-			if err != nil || taskID <= 0 {
-				r.SendBadRequestError(fmt.Errorf("invalid task ID: %s", taskIDStr))
-				return
-			}
-			task, err := replication.OperationCtl.GetTask(taskID)
-			if err != nil {
-				r.SendInternalServerError(fmt.Errorf("failed to get task %d: %v", taskID, err))
-				return
-			}
-			if task == nil || task.ExecutionID != executionID {
-				r.SendNotFoundError(fmt.Errorf("task %d not found", taskID))
-				return
-			}
-			r.task = task
-		}
-	}
-}
-
-// The API is open only for system admin currently, we can use
-// the code commentted below to make the API available to the
-// users who have permission for all projects that the policy
-// refers
-/*
-func (r *ReplicationOperationAPI) authorized(policy *model.Policy, resource rbac.Resource, action rbac.Action) bool {
-
-	projects := []string{}
-	// pull mode
-	if policy.SrcRegistryID != 0 {
-		projects = append(projects, policy.DestNamespace)
-	} else {
-		// push mode
-		projects = append(projects, policy.SrcNamespaces...)
-	}
-
-	for _, project := range projects {
-		resource := rbac.NewProjectNamespace(project).Resource(resource)
-		if !r.SecurityCtx.Can(action, resource) {
-			r.HandleForbidden(r.SecurityCtx.GetUsername())
-			return false
-		}
-	}
-
-	return true
-}
-*/
-
-// ListExecutions ...
-func (r *ReplicationOperationAPI) ListExecutions() {
-	query := &models.ExecutionQuery{
-		Trigger: r.GetString("trigger"),
-	}
-
-	if len(r.GetString("status")) > 0 {
-		query.Statuses = []string{r.GetString("status")}
-	}
-	if len(r.GetString("policy_id")) > 0 {
-		policyID, err := r.GetInt64("policy_id")
-		if err != nil || policyID <= 0 {
-			r.SendBadRequestError(fmt.Errorf("invalid policy_id %s", r.GetString("policy_id")))
-			return
-		}
-		query.PolicyID = policyID
-	}
-	page, size, err := r.GetPaginationParams()
-	if err != nil {
-		r.SendBadRequestError(err)
-		return
-	}
-	query.Page = page
-	query.Size = size
-
-	total, executions, err := replication.OperationCtl.ListExecutions(query)
-	if err != nil {
-		r.SendInternalServerError(fmt.Errorf("failed to list executions: %v", err))
-		return
-	}
-	r.SetPaginationHeader(total, query.Page, query.Size)
-	r.WriteJSONData(executions)
-}
-
-// CreateExecution starts a replication
-func (r *ReplicationOperationAPI) CreateExecution() {
-	execution := &models.Execution{}
-	if err := r.DecodeJSONReq(execution); err != nil {
-		r.SendBadRequestError(err)
-		return
-	}
-
-	policy, err := replication.PolicyCtl.Get(execution.PolicyID)
-	if err != nil {
-		r.SendInternalServerError(fmt.Errorf("failed to get policy %d: %v", execution.PolicyID, err))
-		return
-	}
-	if policy == nil {
-		r.SendNotFoundError(fmt.Errorf("policy %d not found", execution.PolicyID))
-		return
-	}
-	if !policy.Enabled {
-		r.SendBadRequestError(fmt.Errorf("the policy %d is disabled", execution.PolicyID))
-		return
-	}
-	if err = event.PopulateRegistries(replication.RegistryMgr, policy); err != nil {
-		r.SendInternalServerError(fmt.Errorf("failed to populate registries for policy %d: %v", execution.PolicyID, err))
-		return
-	}
-
-	trigger := r.GetString("trigger", string(model.TriggerTypeManual))
-	executionID, err := replication.OperationCtl.StartReplication(policy, nil, model.TriggerType(trigger))
-	if err != nil {
-		r.SendInternalServerError(fmt.Errorf("failed to start replication for policy %d: %v", execution.PolicyID, err))
-		return
-	}
-	r.Redirect(http.StatusCreated, strconv.FormatInt(executionID, 10))
-}
-
-// GetExecution gets one execution of the replication
-func (r *ReplicationOperationAPI) GetExecution() {
-	r.WriteJSONData(r.execution)
-}
-
-// StopExecution stops one execution of the replication
-func (r *ReplicationOperationAPI) StopExecution() {
-	if err := replication.OperationCtl.StopReplication(r.execution.ID); err != nil {
-		r.SendInternalServerError(fmt.Errorf("failed to stop execution %d: %v", r.execution.ID, err))
-		return
-	}
-}
-
-// ListTasks ...
-func (r *ReplicationOperationAPI) ListTasks() {
-	query := &models.TaskQuery{
-		ExecutionID:  r.execution.ID,
-		ResourceType: r.GetString("resource_type"),
-	}
-	status := r.GetString("status")
-	if len(status) > 0 {
-		query.Statuses = []string{status}
-	}
-	page, size, err := r.GetPaginationParams()
-	if err != nil {
-		r.SendBadRequestError(err)
-		return
-	}
-	query.Page = page
-	query.Size = size
-	total, tasks, err := replication.OperationCtl.ListTasks(query)
-	if err != nil {
-		r.SendInternalServerError(fmt.Errorf("failed to list tasks: %v", err))
-		return
-	}
-	r.SetPaginationHeader(total, query.Page, query.Size)
-	r.WriteJSONData(tasks)
-}
-
-// GetTaskLog ...
-func (r *ReplicationOperationAPI) GetTaskLog() {
-	logBytes, err := replication.OperationCtl.GetTaskLog(r.task.ID)
-	if err != nil {
-		if httpErr, ok := err.(*common_http.Error); ok {
-			if ok && httpErr.Code == http.StatusNotFound {
-				r.SendNotFoundError(fmt.Errorf("the log of task %d not found", r.task.ID))
-				return
-			}
-		}
-		r.SendInternalServerError(fmt.Errorf("failed to get log of task %d: %v", r.task.ID, err))
-		return
-	}
-	r.Ctx.ResponseWriter.Header().Set(http.CanonicalHeaderKey("Content-Length"), strconv.Itoa(len(logBytes)))
-	r.Ctx.ResponseWriter.Header().Set(http.CanonicalHeaderKey("Content-Type"), "text/plain")
-	_, err = r.Ctx.ResponseWriter.Write(logBytes)
-	if err != nil {
-		r.SendInternalServerError(fmt.Errorf("failed to write log of task %d: %v", r.task.ID, err))
-		return
-	}
-}

src/core/api/replication_execution_test.go

@@ -1,443 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package api
-
-import (
-	"net/http"
-	"testing"
-
-	"github.com/goharbor/harbor/src/replication"
-	"github.com/goharbor/harbor/src/replication/dao/models"
-	"github.com/goharbor/harbor/src/replication/model"
-)
-
-type fakedOperationController struct{}
-
-func (f *fakedOperationController) StartReplication(policy *model.Policy, resource *model.Resource, trigger model.TriggerType) (int64, error) {
-	return 1, nil
-}
-func (f *fakedOperationController) StopReplication(int64) error {
-	return nil
-}
-func (f *fakedOperationController) ListExecutions(...*models.ExecutionQuery) (int64, []*models.Execution, error) {
-	return 1, []*models.Execution{
-		{
-			ID:       1,
-			PolicyID: 1,
-		},
-	}, nil
-}
-func (f *fakedOperationController) GetExecution(id int64) (*models.Execution, error) {
-	if id == 1 {
-		return &models.Execution{
-			ID:       1,
-			PolicyID: 1,
-		}, nil
-	}
-	return nil, nil
-}
-func (f *fakedOperationController) ListTasks(...*models.TaskQuery) (int64, []*models.Task, error) {
-	return 2, []*models.Task{
-		{
-			ID:          1,
-			ExecutionID: 1,
-		},
-	}, nil
-}
-func (f *fakedOperationController) GetTask(id int64) (*models.Task, error) {
-	if id == 1 {
-		return &models.Task{
-			ID:          1,
-			ExecutionID: 1,
-		}, nil
-	}
-	if id == 3 {
-		return &models.Task{
-			ID:          3,
-			ExecutionID: 2,
-		}, nil
-	}
-	return nil, nil
-}
-func (f *fakedOperationController) UpdateTaskStatus(id int64, status string, statusRevision int64, statusCondition ...string) error {
-	return nil
-}
-func (f *fakedOperationController) GetTaskLog(int64) ([]byte, error) {
-	return []byte("success"), nil
-}
-
-type fakedPolicyManager struct{}
-
-func (f *fakedPolicyManager) Create(*model.Policy) (int64, error) {
-	return 0, nil
-}
-func (f *fakedPolicyManager) List(...*model.PolicyQuery) (int64, []*model.Policy, error) {
-	return 0, nil, nil
-}
-func (f *fakedPolicyManager) Get(id int64) (*model.Policy, error) {
-	if id == 1 {
-		return &model.Policy{
-			ID:      1,
-			Enabled: true,
-			SrcRegistry: &model.Registry{
-				ID: 1,
-			},
-		}, nil
-	}
-	if id == 2 {
-		return &model.Policy{
-			ID:      2,
-			Enabled: false,
-			SrcRegistry: &model.Registry{
-				ID: 1,
-			},
-		}, nil
-	}
-	return nil, nil
-}
-func (f *fakedPolicyManager) GetByName(name string) (*model.Policy, error) {
-	if name == "duplicate_name" {
-		return &model.Policy{
-			Name: "duplicate_name",
-		}, nil
-	}
-	return nil, nil
-}
-func (f *fakedPolicyManager) Update(*model.Policy) error {
-	return nil
-}
-func (f *fakedPolicyManager) Remove(int64) error {
-	return nil
-}
-
-func TestListExecutions(t *testing.T) {
-	operationCtl := replication.OperationCtl
-	defer func() {
-		replication.OperationCtl = operationCtl
-	}()
-	replication.OperationCtl = &fakedOperationController{}
-
-	cases := []*codeCheckingCase{
-		// 401
-		{
-			request: &testingRequest{
-				method: http.MethodGet,
-				url:    "/api/replication/executions",
-			},
-			code: http.StatusUnauthorized,
-		},
-		// 403
-		{
-			request: &testingRequest{
-				method:     http.MethodGet,
-				url:        "/api/replication/executions",
-				credential: nonSysAdmin,
-			},
-			code: http.StatusForbidden,
-		},
-		// 200
-		{
-			request: &testingRequest{
-				method:     http.MethodGet,
-				url:        "/api/replication/executions",
-				credential: sysAdmin,
-			},
-			code: http.StatusOK,
-		},
-	}
-
-	runCodeCheckingCases(t, cases...)
-}
-
-func TestCreateExecution(t *testing.T) {
-	operationCtl := replication.OperationCtl
-	policyMgr := replication.PolicyCtl
-	registryMgr := replication.RegistryMgr
-	defer func() {
-		replication.OperationCtl = operationCtl
-		replication.PolicyCtl = policyMgr
-		replication.RegistryMgr = registryMgr
-	}()
-	replication.OperationCtl = &fakedOperationController{}
-	replication.PolicyCtl = &fakedPolicyManager{}
-	replication.RegistryMgr = &fakedRegistryManager{}
-
-	cases := []*codeCheckingCase{
-		// 401
-		{
-			request: &testingRequest{
-				method: http.MethodPost,
-				url:    "/api/replication/executions",
-			},
-			code: http.StatusUnauthorized,
-		},
-		// 403
-		{
-			request: &testingRequest{
-				method:     http.MethodPost,
-				url:        "/api/replication/executions",
-				credential: nonSysAdmin,
-			},
-			code: http.StatusForbidden,
-		},
-		// 404
-		{
-			request: &testingRequest{
-				method: http.MethodPost,
-				url:    "/api/replication/executions",
-				bodyJSON: &models.Execution{
-					PolicyID: 3,
-				},
-				credential: sysAdmin,
-			},
-			code: http.StatusNotFound,
-		},
-		// 400
-		{
-			request: &testingRequest{
-				method: http.MethodPost,
-				url:    "/api/replication/executions",
-				bodyJSON: &models.Execution{
-					PolicyID: 2,
-				},
-				credential: sysAdmin,
-			},
-			code: http.StatusBadRequest,
-		},
-		// 201
-		{
-			request: &testingRequest{
-				method: http.MethodPost,
-				url:    "/api/replication/executions",
-				bodyJSON: &models.Execution{
-					PolicyID: 1,
-				},
-				credential: sysAdmin,
-			},
-			code: http.StatusCreated,
-		},
-	}
-
-	runCodeCheckingCases(t, cases...)
-}
-
-func TestGetExecution(t *testing.T) {
-	operationCtl := replication.OperationCtl
-	defer func() {
-		replication.OperationCtl = operationCtl
-	}()
-	replication.OperationCtl = &fakedOperationController{}
-
-	cases := []*codeCheckingCase{
-		// 401
-		{
-			request: &testingRequest{
-				method: http.MethodGet,
-				url:    "/api/replication/executions/1",
-			},
-			code: http.StatusUnauthorized,
-		},
-		// 403
-		{
-			request: &testingRequest{
-				method:     http.MethodGet,
-				url:        "/api/replication/executions/1",
-				credential: nonSysAdmin,
-			},
-			code: http.StatusForbidden,
-		},
-		// 404
-		{
-			request: &testingRequest{
-				method:     http.MethodGet,
-				url:        "/api/replication/executions/2",
-				credential: sysAdmin,
-			},
-			code: http.StatusNotFound,
-		},
-		// 200
-		{
-			request: &testingRequest{
-				method:     http.MethodGet,
-				url:        "/api/replication/executions/1",
-				credential: sysAdmin,
-			},
-			code: http.StatusOK,
-		},
-	}
-
-	runCodeCheckingCases(t, cases...)
-}
-func TestStopExecution(t *testing.T) {
-	operationCtl := replication.OperationCtl
-	defer func() {
-		replication.OperationCtl = operationCtl
-	}()
-	replication.OperationCtl = &fakedOperationController{}
-
-	cases := []*codeCheckingCase{
-		// 401
-		{
-			request: &testingRequest{
-				method: http.MethodPut,
-				url:    "/api/replication/executions/1",
-			},
-			code: http.StatusUnauthorized,
-		},
-		// 403
-		{
-			request: &testingRequest{
-				method:     http.MethodPut,
-				url:        "/api/replication/executions/1",
-				credential: nonSysAdmin,
-			},
-			code: http.StatusForbidden,
-		},
-		// 404
-		{
-			request: &testingRequest{
-				method:     http.MethodPut,
-				url:        "/api/replication/executions/2",
-				credential: sysAdmin,
-			},
-			code: http.StatusNotFound,
-		},
-		// 200
-		{
-			request: &testingRequest{
-				method:     http.MethodPut,
-				url:        "/api/replication/executions/1",
-				credential: sysAdmin,
-			},
-			code: http.StatusOK,
-		},
-	}
-
-	runCodeCheckingCases(t, cases...)
-}
-
-func TestListTasks(t *testing.T) {
-	operationCtl := replication.OperationCtl
-	defer func() {
-		replication.OperationCtl = operationCtl
-	}()
-	replication.OperationCtl = &fakedOperationController{}
-
-	cases := []*codeCheckingCase{
-		// 401
-		{
-			request: &testingRequest{
-				method: http.MethodGet,
-				url:    "/api/replication/executions/1/tasks",
-			},
-			code: http.StatusUnauthorized,
-		},
-		// 403
-		{
-			request: &testingRequest{
-				method:     http.MethodGet,
-				url:        "/api/replication/executions/1/tasks",
-				credential: nonSysAdmin,
-			},
-			code: http.StatusForbidden,
-		},
-		// 404
-		{
-			request: &testingRequest{
-				method:     http.MethodGet,
-				url:        "/api/replication/executions/2/tasks",
-				credential: sysAdmin,
-			},
-			code: http.StatusNotFound,
-		},
-		// 200
-		{
-			request: &testingRequest{
-				method:     http.MethodGet,
-				url:        "/api/replication/executions/1/tasks",
-				credential: sysAdmin,
-			},
-			code: http.StatusOK,
-		},
-	}
-
-	runCodeCheckingCases(t, cases...)
-}
-
-func TestGetTaskLog(t *testing.T) {
-	operationCtl := replication.OperationCtl
-	defer func() {
-		replication.OperationCtl = operationCtl
-	}()
-	replication.OperationCtl = &fakedOperationController{}
-
-	cases := []*codeCheckingCase{
-		// 401
-		{
-			request: &testingRequest{
-				method: http.MethodGet,
-				url:    "/api/replication/executions/1/tasks/1/log",
-			},
-			code: http.StatusUnauthorized,
-		},
-		// 403
-		{
-			request: &testingRequest{
-				method:     http.MethodGet,
-				url:        "/api/replication/executions/1/tasks/1/log",
-				credential: nonSysAdmin,
-			},
-			code: http.StatusForbidden,
-		},
-		// 404, execution not found
-		{
-			request: &testingRequest{
-				method:     http.MethodGet,
-				url:        "/api/replication/executions/2/tasks/1/log",
-				credential: sysAdmin,
-			},
-			code: http.StatusNotFound,
-		},
-		// 404, task not found
-		{
-			request: &testingRequest{
-				method:     http.MethodGet,
-				url:        "/api/replication/executions/1/tasks/2/log",
-				credential: sysAdmin,
-			},
-			code: http.StatusNotFound,
-		},
-		// 404, task not found
-		{
-			request: &testingRequest{
-				method:     http.MethodGet,
-				url:        "/api/replication/executions/1/tasks/3/log",
-				credential: sysAdmin,
-			},
-			code: http.StatusNotFound,
-		},
-		// 200
-		{
-			request: &testingRequest{
-				method:     http.MethodGet,
-				url:        "/api/replication/executions/1/tasks/1/log",
-				credential: sysAdmin,
-			},
-			code: http.StatusOK,
-		},
-	}
-
-	runCodeCheckingCases(t, cases...)
-}

src/core/api/replication_policy.go

@@ -17,11 +17,15 @@ package api
 import (
 	"errors"
 	"fmt"
+	"github.com/goharbor/harbor/src/jobservice/job"
+	"github.com/goharbor/harbor/src/lib/orm"
+	"github.com/goharbor/harbor/src/lib/q"
 	"net/http"
 	"strconv"
 
+	replica "github.com/goharbor/harbor/src/controller/replication"
+	"github.com/goharbor/harbor/src/pkg/task"
 	"github.com/goharbor/harbor/src/replication"
-	"github.com/goharbor/harbor/src/replication/dao/models"
 	"github.com/goharbor/harbor/src/replication/event"
 	"github.com/goharbor/harbor/src/replication/model"
 	"github.com/goharbor/harbor/src/replication/registry"
@@ -219,16 +223,29 @@ func (r *ReplicationPolicyAPI) Delete() {
 		return
 	}
 
-	isRunning, err := hasRunningExecutions(id)
+	ctx := orm.Context()
+	executions, err := replica.Ctl.ListExecutions(ctx, &q.Query{
+		Keywords: map[string]interface{}{
+			"PolicyID": id,
+		},
+	})
 	if err != nil {
-		r.SendInternalServerError(fmt.Errorf("failed to check the execution status of policy %d: %v", id, err))
+		r.SendInternalServerError(err)
 		return
 	}
-
+	for _, execution := range executions {
-	if isRunning {
+		if execution.Status != job.RunningStatus.String() {
+			continue
+		}
 		r.SendPreconditionFailedError(fmt.Errorf("the policy %d has running executions, can not be deleted", id))
 		return
 	}
+	for _, execution := range executions {
+		if err = task.ExecMgr.Delete(ctx, execution.ID); err != nil {
+			r.SendInternalServerError(err)
+			return
+		}
+	}
 
 	if err := replication.PolicyCtl.Remove(id); err != nil {
 		r.SendInternalServerError(fmt.Errorf("failed to delete the policy %d: %v", id, err))
@@ -236,22 +253,6 @@ func (r *ReplicationPolicyAPI) Delete() {
 	}
 }
 
-func hasRunningExecutions(policyID int64) (bool, error) {
-	_, executions, err := replication.OperationCtl.ListExecutions(&models.ExecutionQuery{
-		PolicyID: policyID,
-	})
-	if err != nil {
-		return false, err
-	}
-	for _, execution := range executions {
-		if execution.Status != models.ExecutionStatusInProgress {
-			continue
-		}
-		return true, nil
-	}
-	return false, nil
-}
-
 // ignore the credential for the registries
 func populateRegistries(registryMgr registry.Manager, policy *model.Policy) error {
 	if err := event.PopulateRegistries(registryMgr, policy); err != nil {

src/core/api/replication_policy_test.go

@@ -54,6 +54,50 @@ func (f *fakedRegistryManager) HealthCheck() error {
 	return nil
 }
 
+type fakedPolicyManager struct{}
+
+func (f *fakedPolicyManager) Create(*model.Policy) (int64, error) {
+	return 0, nil
+}
+func (f *fakedPolicyManager) List(...*model.PolicyQuery) (int64, []*model.Policy, error) {
+	return 0, nil, nil
+}
+func (f *fakedPolicyManager) Get(id int64) (*model.Policy, error) {
+	if id == 1 {
+		return &model.Policy{
+			ID:      1,
+			Enabled: true,
+			SrcRegistry: &model.Registry{
+				ID: 1,
+			},
+		}, nil
+	}
+	if id == 2 {
+		return &model.Policy{
+			ID:      2,
+			Enabled: false,
+			SrcRegistry: &model.Registry{
+				ID: 1,
+			},
+		}, nil
+	}
+	return nil, nil
+}
+func (f *fakedPolicyManager) GetByName(name string) (*model.Policy, error) {
+	if name == "duplicate_name" {
+		return &model.Policy{
+			Name: "duplicate_name",
+		}, nil
+	}
+	return nil, nil
+}
+func (f *fakedPolicyManager) Update(*model.Policy) error {
+	return nil
+}
+func (f *fakedPolicyManager) Remove(int64) error {
+	return nil
+}
+
 func TestReplicationPolicyAPIList(t *testing.T) {
 	policyMgr := replication.PolicyCtl
 	defer func() {

src/core/service/notifications/jobs/handler.go

@@ -31,9 +31,6 @@ import (
 	"github.com/goharbor/harbor/src/pkg/notifier/event"
 	"github.com/goharbor/harbor/src/pkg/retention"
 	sc "github.com/goharbor/harbor/src/pkg/scan"
-	"github.com/goharbor/harbor/src/replication"
-	"github.com/goharbor/harbor/src/replication/operation/hook"
-	"github.com/goharbor/harbor/src/replication/policy/scheduler"
 )
 
 var statusMap = map[string]string{
@@ -141,44 +138,6 @@ func (h *Handler) HandleScan() {
 	}
 }
 
-// HandleReplicationScheduleJob handles the webhook of replication schedule job
-func (h *Handler) HandleReplicationScheduleJob() {
-	log.Debugf("received replication schedule job status update event: schedule-job-%d, status-%s", h.id, h.status)
-	if err := scheduler.UpdateStatus(h.id, h.status); err != nil {
-		log.Errorf("Failed to update job status, id: %d, status: %s", h.id, h.status)
-		h.SendInternalServerError(err)
-		return
-	}
-}
-
-// HandleReplicationTask handles the webhook of replication task
-func (h *Handler) HandleReplicationTask() {
-	log.Debugf("received replication task status update event: task-%d, status-%s", h.id, h.status)
-
-	if err := hook.UpdateTask(replication.OperationCtl, h.id, h.rawStatus, h.revision); err != nil {
-		log.Errorf("failed to update the status of the replication task %d: %v", h.id, err)
-		h.SendInternalServerError(err)
-		return
-	}
-
-	// Trigger artifict webhook event only for JobFinished and JobError status
-	if h.status == models.JobFinished || h.status == models.JobError || h.status == models.JobStopped {
-		e := &event.Event{}
-		metaData := &metadata.ReplicationMetaData{
-			ReplicationTaskID: h.id,
-			Status:            h.rawStatus,
-		}
-
-		if err := e.Build(metaData); err == nil {
-			if err := e.Publish(); err != nil {
-				log.Error(errors.Wrap(err, "replication job hook handler: event publish"))
-			}
-		} else {
-			log.Error(errors.Wrap(err, "replication job hook handler: event publish"))
-		}
-	}
-}
-
 // HandleRetentionTask handles the webhook of retention task
 func (h *Handler) HandleRetentionTask() {
 	taskID := h.id

src/core/service/notifications/scheduler/handler.go

@@ -1,51 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package scheduler
-
-import (
-	"encoding/json"
-
-	"github.com/goharbor/harbor/src/core/service/notifications"
-	"github.com/goharbor/harbor/src/jobservice/job"
-	"github.com/goharbor/harbor/src/lib/log"
-	"github.com/goharbor/harbor/src/pkg/scheduler"
-)
-
-// Handler handles the scheduler requests
-type Handler struct {
-	notifications.BaseHandler
-}
-
-// Handle ...
-func (h *Handler) Handle() {
-	log.Debugf("received scheduler hook event for schedule %s", h.GetStringFromPath(":id"))
-
-	var data job.StatusChange
-	if err := json.Unmarshal(h.Ctx.Input.CopyBody(1<<32), &data); err != nil {
-		log.Errorf("failed to decode hook event: %v", err)
-		return
-	}
-
-	schedulerID, err := h.GetInt64FromPath(":id")
-	if err != nil {
-		log.Errorf("failed to get the schedule ID: %v", err)
-		return
-	}
-
-	if err = scheduler.HandleLegacyHook(h.Ctx.Request.Context(), schedulerID, &data); err != nil {
-		log.Errorf("failed to handle the legacy hook: %v", err)
-		return
-	}
-}

src/lib/log/context.go

@@ -35,8 +35,11 @@ func WithLogger(ctx context.Context, logger *Logger) context.Context {
 // GetLogger retrieves the current logger from the context.
 // If no logger is available, the default logger is returned.
 func GetLogger(ctx context.Context) *Logger {
-	logger := ctx.Value(loggerKey{})
+	if ctx == nil {
+		return L
+	}
 
+	logger := ctx.Value(loggerKey{})
 	if logger == nil {
 		return L
 	}

src/lib/orm/creator.go

@@ -12,25 +12,28 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package flow
+package orm
 
-import (
+import "github.com/astaxie/beego/orm"
-	"testing"
 
-	"github.com/stretchr/testify/assert"
+var (
-	"github.com/stretchr/testify/require"
+	// Crt is a global instance of ORM creator
+	Crt = NewCreator()
 )
 
-type fakedFlow struct{}
+// NewCreator creates an ORM creator
-
+func NewCreator() Creator {
-func (f *fakedFlow) Run(interface{}) (int, error) {
+	return &creator{}
-	return 1, nil
 }
 
-func TestStart(t *testing.T) {
+// Creator creates ORMer
-	flow := &fakedFlow{}
+// Introducing the "Creator" interface to eliminate the dependency on database
-	controller := NewController()
+type Creator interface {
-	n, err := controller.Start(flow)
+	Create() orm.Ormer
-	require.Nil(t, err)
+}
-	assert.Equal(t, 1, n)
+
+type creator struct{}
+
+func (c *creator) Create() orm.Ormer {
+	return orm.NewOrm()
 }

src/lib/q/query.go

@@ -37,11 +37,18 @@ func New(kw KeyWords) *Query {
 // MustClone returns the clone of query when it's not nil
 // or returns a new Query instance
 func MustClone(query *Query) *Query {
-	if query != nil {
+	q := &Query{
-		clone := *query
+		Keywords: map[string]interface{}{},
-		return &clone
 	}
-	return New(KeyWords{})
+	if query != nil {
+		q.PageNumber = query.PageNumber
+		q.PageSize = query.PageSize
+		q.Sorting = query.Sorting
+		for k, v := range query.Keywords {
+			q.Keywords[k] = v
+		}
+	}
+	return q
 }
 
 // Range query

src/lib/worker_pool.go

@@ -12,26 +12,26 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package flow
+package lib
 
-// Flow defines the replication flow
+// NewWorkerPool creates a new worker pool with specified size
-type Flow interface {
+func NewWorkerPool(size int32) *WorkerPool {
-	// returns the count of tasks which have been scheduled and the error
+	wp := &WorkerPool{}
-	Run(interface{}) (int, error)
+	wp.queue = make(chan struct{}, size)
+	return wp
 }
 
-// Controller is the controller that controls the replication flows
+// WorkerPool controls the concurrency limit of task/process
-type Controller interface {
+type WorkerPool struct {
-	Start(Flow) (int, error)
+	queue chan struct{}
 }
 
-// NewController returns an instance of the default flow controller
+// GetWorker hangs until a free worker available
-func NewController() Controller {
+func (w *WorkerPool) GetWorker() {
-	return &controller{}
+	w.queue <- struct{}{}
 }
 
-type controller struct{}
+// ReleaseWorker hangs until the worker return back into the pool
-
+func (w *WorkerPool) ReleaseWorker() {
-func (c *controller) Start(flow Flow) (int, error) {
+	<-w.queue
-	return flow.Run(nil)
 }

src/pkg/scheduler/callback.go

@@ -18,7 +18,6 @@ import (
 	"context"
 	"fmt"
 
-	"github.com/goharbor/harbor/src/jobservice/job"
 	"github.com/goharbor/harbor/src/lib/errors"
 	"github.com/goharbor/harbor/src/lib/log"
 	"github.com/goharbor/harbor/src/pkg/task"
@@ -29,7 +28,7 @@ var (
 )
 
 // CallbackFunc defines the function that the scheduler calls when triggered
-type CallbackFunc func(interface{}) error
+type CallbackFunc func(ctx context.Context, param string) error
 
 func init() {
 	if err := task.RegisterCheckInProcessor(JobNameScheduler, triggerCallback); err != nil {
@@ -68,7 +67,7 @@ func callbackFuncExist(name string) bool {
 	return exist
 }
 
-func triggerCallback(ctx context.Context, task *task.Task, change *job.StatusChange) (err error) {
+func triggerCallback(ctx context.Context, task *task.Task, data string) (err error) {
 	execution, err := Sched.(*scheduler).execMgr.Get(ctx, task.ExecutionID)
 	if err != nil {
 		return err
@@ -85,5 +84,5 @@ func triggerCallback(ctx context.Context, task *task.Task, change *job.StatusCha
 	if err != nil {
 		return err
 	}
-	return callbackFunc(schedule.CallbackFuncParam)
+	return callbackFunc(ctx, schedule.CallbackFuncParam)
 }

src/pkg/scheduler/callback_test.go

@@ -15,6 +15,7 @@
 package scheduler
 
 import (
+	"context"
 	"testing"
 
 	"github.com/stretchr/testify/suite"
@@ -26,7 +27,7 @@ type callbackTestSuite struct {
 
 func (c *callbackTestSuite) SetupTest() {
 	registry = map[string]CallbackFunc{}
-	err := RegisterCallbackFunc("callback", func(interface{}) error { return nil })
+	err := RegisterCallbackFunc("callback", func(context.Context, string) error { return nil })
 	c.Require().Nil(err)
 }
 
@@ -40,11 +41,11 @@ func (c *callbackTestSuite) TestRegisterCallbackFunc() {
 	c.NotNil(err)
 
 	// pass
-	err = RegisterCallbackFunc("test", func(interface{}) error { return nil })
+	err = RegisterCallbackFunc("test", func(context.Context, string) error { return nil })
 	c.Nil(err)
 
 	// duplicate name
-	err = RegisterCallbackFunc("test", func(interface{}) error { return nil })
+	err = RegisterCallbackFunc("test", func(context.Context, string) error { return nil })
 	c.NotNil(err)
 }
 

src/pkg/scheduler/scheduler.go

@@ -289,38 +289,3 @@ func (s *scheduler) convertSchedule(ctx context.Context, schedule *schedule) (*S
 	}
 	return schd, nil
 }
-
-// HandleLegacyHook handles the legacy web hook for scheduler
-// We rewrite the implementation of scheduler with task manager mechanism in v2.1,
-// this method is used to handle the job status hook for the legacy implementation
-// We can remove the method and the hook endpoint after several releases
-func HandleLegacyHook(ctx context.Context, scheduleID int64, sc *job.StatusChange) error {
-	scheduler := Sched.(*scheduler)
-	executions, err := scheduler.execMgr.List(ctx, &q.Query{
-		Keywords: map[string]interface{}{
-			"VendorType": JobNameScheduler,
-			"VendorID":   scheduleID,
-		},
-	})
-	if err != nil {
-		return err
-	}
-	if len(executions) == 0 {
-		return errors.New(nil).WithCode(errors.NotFoundCode).
-			WithMessage("no execution found for the schedule %d", scheduleID)
-	}
-
-	tasks, err := scheduler.taskMgr.List(ctx, &q.Query{
-		Keywords: map[string]interface{}{
-			"ExecutionID": executions[0].ID,
-		},
-	})
-	if err != nil {
-		return err
-	}
-	if len(tasks) == 0 {
-		return errors.New(nil).WithCode(errors.NotFoundCode).
-			WithMessage("no task found for the execution %d", executions[0].ID)
-	}
-	return task.NewHookHandler().Handle(ctx, tasks[0].ID, sc)
-}

src/pkg/scheduler/scheduler_test.go

@@ -15,13 +15,15 @@
 package scheduler
 
 import (
+	"context"
 	"fmt"
+	"testing"
+
 	"github.com/goharbor/harbor/src/jobservice/job"
 	"github.com/goharbor/harbor/src/pkg/task"
 	"github.com/goharbor/harbor/src/testing/mock"
 	tasktesting "github.com/goharbor/harbor/src/testing/pkg/task"
 	"github.com/stretchr/testify/suite"
-	"testing"
 )
 
 type schedulerTestSuite struct {
@@ -34,7 +36,7 @@ type schedulerTestSuite struct {
 
 func (s *schedulerTestSuite) SetupTest() {
 	registry = map[string]CallbackFunc{}
-	err := RegisterCallbackFunc("callback", func(interface{}) error { return nil })
+	err := RegisterCallbackFunc("callback", func(context.Context, string) error { return nil })
 	s.Require().Nil(err)
 
 	s.dao = &mockDAO{}

src/pkg/task/checkin.go

@@ -1,38 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package task
-
-import (
-	"context"
-	"fmt"
-
-	"github.com/goharbor/harbor/src/jobservice/job"
-)
-
-var (
-	registry = map[string]CheckInProcessor{}
-)
-
-// CheckInProcessor is the processor to process the check in data which is sent by jobservice via webhook
-type CheckInProcessor func(ctx context.Context, task *Task, change *job.StatusChange) (err error)
-
-// RegisterCheckInProcessor registers check in processor for the specific vendor type
-func RegisterCheckInProcessor(vendorType string, processor CheckInProcessor) error {
-	if _, exist := registry[vendorType]; exist {
-		return fmt.Errorf("check in processor for %s already exists", vendorType)
-	}
-	registry[vendorType] = processor
-	return nil
-}

src/pkg/task/dao/execution.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"fmt"
 	"github.com/goharbor/harbor/src/lib/log"
+	"strings"
 
 	"github.com/goharbor/harbor/src/jobservice/job"
 	"github.com/goharbor/harbor/src/lib/errors"
@@ -28,8 +29,10 @@ import (
 // ExecutionDAO is the data access object interface for execution
 type ExecutionDAO interface {
 	// Count returns the total count of executions according to the query
+	// Query the "ExtraAttrs" by setting 'query.Keywords["ExtraAttrs.key"]="value"'
 	Count(ctx context.Context, query *q.Query) (count int64, err error)
 	// List the executions according to the query
+	// Query the "ExtraAttrs" by setting 'query.Keywords["ExtraAttrs.key"]="value"'
 	List(ctx context.Context, query *q.Query) (executions []*Execution, err error)
 	// Get the specified execution
 	Get(ctx context.Context, id int64) (execution *Execution, err error)
@@ -43,7 +46,9 @@ type ExecutionDAO interface {
 	GetMetrics(ctx context.Context, id int64) (metrics *Metrics, err error)
 	// RefreshStatus refreshes the status of the specified execution according to it's tasks. If it's status
 	// is final, update the end time as well
-	RefreshStatus(ctx context.Context, id int64) (err error)
+	// If the status is changed, the returning "statusChanged" is set as "true" and the current status indicates
+	// the changed status
+	RefreshStatus(ctx context.Context, id int64) (statusChanged bool, currentStatus string, err error)
 }
 
 // NewExecutionDAO returns an instance of ExecutionDAO
@@ -64,7 +69,7 @@ func (e *executionDAO) Count(ctx context.Context, query *q.Query) (int64, error)
 			Keywords: query.Keywords,
 		}
 	}
-	qs, err := orm.QuerySetter(ctx, &Execution{}, query)
+	qs, err := e.querySetter(ctx, query)
 	if err != nil {
 		return 0, err
 	}
@@ -73,7 +78,7 @@ func (e *executionDAO) Count(ctx context.Context, query *q.Query) (int64, error)
 
 func (e *executionDAO) List(ctx context.Context, query *q.Query) ([]*Execution, error) {
 	executions := []*Execution{}
-	qs, err := orm.QuerySetter(ctx, &Execution{}, query)
+	qs, err := e.querySetter(ctx, query)
 	if err != nil {
 		return nil, err
 	}
@@ -178,33 +183,39 @@ func (e *executionDAO) GetMetrics(ctx context.Context, id int64) (*Metrics, erro
 		metrics.ScheduledTaskCount + metrics.StoppedTaskCount
 	return metrics, nil
 }
-func (e *executionDAO) RefreshStatus(ctx context.Context, id int64) error {
+
+func (e *executionDAO) RefreshStatus(ctx context.Context, id int64) (bool, string, error) {
 	// as the status of the execution can be refreshed by multiple operators concurrently
 	// we use the optimistic locking to avoid the conflict and retry 5 times at most
 	for i := 0; i < 5; i++ {
-		retry, err := e.refreshStatus(ctx, id)
+		statusChanged, currentStatus, retry, err := e.refreshStatus(ctx, id)
 		if err != nil {
-			return err
+			return false, "", err
 		}
 		if !retry {
-			return nil
+			return statusChanged, currentStatus, nil
 		}
 	}
-	return fmt.Errorf("failed to refresh the status of the execution %d after %d retries", id, 5)
+	return false, "", fmt.Errorf("failed to refresh the status of the execution %d after %d retries", id, 5)
 }
 
-func (e *executionDAO) refreshStatus(ctx context.Context, id int64) (bool, error) {
+// the returning values:
+// 1. bool: is the status changed
+// 2. string: the current status if changed
+// 3. bool: whether a retry is needed
+// 4. error: the error
+func (e *executionDAO) refreshStatus(ctx context.Context, id int64) (bool, string, bool, error) {
 	execution, err := e.Get(ctx, id)
 	if err != nil {
-		return false, err
+		return false, "", false, err
 	}
 	metrics, err := e.GetMetrics(ctx, id)
 	if err != nil {
-		return false, err
+		return false, "", false, err
 	}
 	// no task, return directly
 	if metrics.TaskCount == 0 {
-		return false, nil
+		return false, "", false, nil
 	}
 
 	var status string
@@ -220,20 +231,22 @@ func (e *executionDAO) refreshStatus(ctx context.Context, id int64) (bool, error
 
 	ormer, err := orm.FromContext(ctx)
 	if err != nil {
-		return false, err
+		return false, "", false, err
 	}
+
 	sql := `update execution set status = ?, revision = revision+1 where id = ? and revision = ?`
 	result, err := ormer.Raw(sql, status, id, execution.Revision).Exec()
 	if err != nil {
-		return false, err
+		return false, "", false, err
 	}
 	n, err := result.RowsAffected()
 	if err != nil {
-		return false, err
+		return false, "", false, err
 	}
+
 	// if the count of affected rows is 0, that means the execution is updating by others, retry
 	if n == 0 {
-		return true, nil
+		return false, "", true, nil
 	}
 
 	/* this is another solution to solve the concurrency issue for refreshing the execution status
@@ -290,5 +303,28 @@ func (e *executionDAO) refreshStatus(ctx context.Context, id int64) (bool, error
 			where id=?`
 	sql = fmt.Sprintf(sql, job.ErrorStatus.String(), job.StoppedStatus.String(), job.SuccessStatus.String())
 	_, err = ormer.Raw(sql, id, id).Exec()
-	return false, err
+	return status != execution.Status, status, false, err
+}
+
+func (e *executionDAO) querySetter(ctx context.Context, query *q.Query) (orm.QuerySeter, error) {
+	qs, err := orm.QuerySetter(ctx, &Execution{}, query)
+	if err != nil {
+		return nil, err
+	}
+
+	// append the filter for "extra attrs"
+	if query != nil && len(query.Keywords) > 0 {
+		for key, value := range query.Keywords {
+			if strings.HasPrefix(key, "ExtraAttrs.") {
+				qs = qs.FilterRaw("id", fmt.Sprintf("in (select id from execution where extra_attrs->>'%s'='%s')", strings.TrimPrefix(key, "ExtraAttrs."), value))
+				break
+			}
+			if strings.HasPrefix(key, "extra_attrs.") {
+				qs = qs.FilterRaw("id", fmt.Sprintf("in (select id from execution where extra_attrs->>'%s'='%s')", strings.TrimPrefix(key, "extra_attrs."), value))
+				break
+			}
+		}
+	}
+
+	return qs, nil
 }

src/pkg/task/dao/execution_test.go

@@ -48,7 +48,7 @@ func (e *executionDAOTestSuite) SetupTest() {
 	id, err := e.executionDAO.Create(e.ctx, &Execution{
 		VendorType: "test",
 		Trigger:    "test",
-		ExtraAttrs: "{}",
+		ExtraAttrs: `{"key":"value"}`,
 	})
 	e.Require().Nil(err)
 	e.executionID = id
@@ -63,21 +63,41 @@ func (e *executionDAOTestSuite) TestCount() {
 	count, err := e.executionDAO.Count(e.ctx, &q.Query{
 		Keywords: map[string]interface{}{
 			"VendorType":     "test",
+			"ExtraAttrs.key": "value",
 		},
 	})
 	e.Require().Nil(err)
 	e.Equal(int64(1), count)
+
+	count, err = e.executionDAO.Count(e.ctx, &q.Query{
+		Keywords: map[string]interface{}{
+			"VendorType":     "test",
+			"ExtraAttrs.key": "incorrect-value",
+		},
+	})
+	e.Require().Nil(err)
+	e.Equal(int64(0), count)
 }
 
 func (e *executionDAOTestSuite) TestList() {
 	executions, err := e.executionDAO.List(e.ctx, &q.Query{
 		Keywords: map[string]interface{}{
 			"VendorType":     "test",
+			"ExtraAttrs.key": "value",
 		},
 	})
 	e.Require().Nil(err)
 	e.Require().Len(executions, 1)
 	e.Equal(e.executionID, executions[0].ID)
+
+	executions, err = e.executionDAO.List(e.ctx, &q.Query{
+		Keywords: map[string]interface{}{
+			"VendorType":     "test",
+			"ExtraAttrs.key": "incorrect-value",
+		},
+	})
+	e.Require().Nil(err)
+	e.Require().Len(executions, 0)
 }
 
 func (e *executionDAOTestSuite) TestGet() {
@@ -200,8 +220,10 @@ func (e *executionDAOTestSuite) TestRefreshStatus() {
 	e.Require().Nil(err)
 	defer e.taskDao.Delete(e.ctx, taskID01)
 
-	err = e.executionDAO.RefreshStatus(e.ctx, e.executionID)
+	statusChanged, currentStatus, err := e.executionDAO.RefreshStatus(e.ctx, e.executionID)
 	e.Require().Nil(err)
+	e.True(statusChanged)
+	e.Equal(job.SuccessStatus.String(), currentStatus)
 	execution, err := e.executionDAO.Get(e.ctx, e.executionID)
 	e.Require().Nil(err)
 	e.Equal(job.SuccessStatus.String(), execution.Status)
@@ -218,8 +240,10 @@ func (e *executionDAOTestSuite) TestRefreshStatus() {
 	e.Require().Nil(err)
 	defer e.taskDao.Delete(e.ctx, taskID02)
 
-	err = e.executionDAO.RefreshStatus(e.ctx, e.executionID)
+	statusChanged, currentStatus, err = e.executionDAO.RefreshStatus(e.ctx, e.executionID)
 	e.Require().Nil(err)
+	e.True(statusChanged)
+	e.Equal(job.StoppedStatus.String(), currentStatus)
 	execution, err = e.executionDAO.Get(e.ctx, e.executionID)
 	e.Require().Nil(err)
 	e.Equal(job.StoppedStatus.String(), execution.Status)
@@ -236,8 +260,10 @@ func (e *executionDAOTestSuite) TestRefreshStatus() {
 	e.Require().Nil(err)
 	defer e.taskDao.Delete(e.ctx, taskID03)
 
-	err = e.executionDAO.RefreshStatus(e.ctx, e.executionID)
+	statusChanged, currentStatus, err = e.executionDAO.RefreshStatus(e.ctx, e.executionID)
 	e.Require().Nil(err)
+	e.True(statusChanged)
+	e.Equal(job.ErrorStatus.String(), currentStatus)
 	execution, err = e.executionDAO.Get(e.ctx, e.executionID)
 	e.Require().Nil(err)
 	e.Equal(job.ErrorStatus.String(), execution.Status)
@@ -271,8 +297,29 @@ func (e *executionDAOTestSuite) TestRefreshStatus() {
 	e.Require().Nil(err)
 	defer e.taskDao.Delete(e.ctx, taskID06)
 
-	err = e.executionDAO.RefreshStatus(e.ctx, e.executionID)
+	statusChanged, currentStatus, err = e.executionDAO.RefreshStatus(e.ctx, e.executionID)
 	e.Require().Nil(err)
+	e.True(statusChanged)
+	e.Equal(job.RunningStatus.String(), currentStatus)
+	execution, err = e.executionDAO.Get(e.ctx, e.executionID)
+	e.Require().Nil(err)
+	e.Equal(job.RunningStatus.String(), execution.Status)
+	e.Empty(execution.EndTime)
+
+	// add another running task, the status shouldn't be changed
+	taskID07, err := e.taskDao.Create(e.ctx, &Task{
+		ExecutionID: e.executionID,
+		Status:      job.RunningStatus.String(),
+		StatusCode:  job.RunningStatus.Code(),
+		ExtraAttrs:  "{}",
+	})
+	e.Require().Nil(err)
+	defer e.taskDao.Delete(e.ctx, taskID07)
+
+	statusChanged, currentStatus, err = e.executionDAO.RefreshStatus(e.ctx, e.executionID)
+	e.Require().Nil(err)
+	e.False(statusChanged)
+	e.Equal(job.RunningStatus.String(), currentStatus)
 	execution, err = e.executionDAO.Get(e.ctx, e.executionID)
 	e.Require().Nil(err)
 	e.Equal(job.RunningStatus.String(), execution.Status)

src/pkg/task/dao/model.go

@@ -56,13 +56,14 @@ type Metrics struct {
 // Task database model
 type Task struct {
 	ID             int64     `orm:"pk;auto;column(id)"`
+	VendorType     string    `orm:"column(vendor_type)"`
 	ExecutionID    int64     `orm:"column(execution_id)"`
 	JobID          string    `orm:"column(job_id)"`
 	Status         string    `orm:"column(status)"`
 	StatusCode     int       `orm:"column(status_code)"`
 	StatusRevision int64     `orm:"column(status_revision)"`
 	StatusMessage  string    `orm:"column(status_message)"`
-	RunCount       int       `orm:"column(run_count)"`
+	RunCount       int32     `orm:"column(run_count)"`
 	ExtraAttrs     string    `orm:"column(extra_attrs)"` // json string
 	CreationTime   time.Time `orm:"column(creation_time)"`
 	StartTime      time.Time `orm:"column(start_time)"`

src/pkg/task/dao/task.go

@@ -16,6 +16,8 @@ package dao
 
 import (
 	"context"
+	"fmt"
+	"strings"
 	"time"
 
 	"github.com/goharbor/harbor/src/jobservice/job"
@@ -27,8 +29,10 @@ import (
 // TaskDAO is the data access object interface for task
 type TaskDAO interface {
 	// Count returns the total count of tasks according to the query
+	// Query the "ExtraAttrs" by setting 'query.Keywords["ExtraAttrs.key"]="value"'
 	Count(ctx context.Context, query *q.Query) (count int64, err error)
 	// List the tasks according to the query
+	// Query the "ExtraAttrs" by setting 'query.Keywords["ExtraAttrs.key"]="value"'
 	List(ctx context.Context, query *q.Query) (tasks []*Task, err error)
 	// Get the specified task
 	Get(ctx context.Context, id int64) (task *Task, err error)
@@ -60,7 +64,7 @@ func (t *taskDAO) Count(ctx context.Context, query *q.Query) (int64, error) {
 			Keywords: query.Keywords,
 		}
 	}
-	qs, err := orm.QuerySetter(ctx, &Task{}, query)
+	qs, err := t.querySetter(ctx, query)
 	if err != nil {
 		return 0, err
 	}
@@ -69,7 +73,7 @@ func (t *taskDAO) Count(ctx context.Context, query *q.Query) (int64, error) {
 
 func (t *taskDAO) List(ctx context.Context, query *q.Query) ([]*Task, error) {
 	tasks := []*Task{}
-	qs, err := orm.QuerySetter(ctx, &Task{}, query)
+	qs, err := t.querySetter(ctx, query)
 	if err != nil {
 		return nil, err
 	}
@@ -203,3 +207,26 @@ func (t *taskDAO) GetMaxEndTime(ctx context.Context, executionID int64) (time.Ti
 		QueryRow(&endTime)
 	return endTime, nil
 }
+
+func (t *taskDAO) querySetter(ctx context.Context, query *q.Query) (orm.QuerySeter, error) {
+	qs, err := orm.QuerySetter(ctx, &Task{}, query)
+	if err != nil {
+		return nil, err
+	}
+
+	// append the filter for "extra attrs"
+	if query != nil && len(query.Keywords) > 0 {
+		for key, value := range query.Keywords {
+			if strings.HasPrefix(key, "ExtraAttrs.") {
+				qs = qs.FilterRaw("id", fmt.Sprintf("in (select id from task where extra_attrs->>'%s'='%s')", strings.TrimPrefix(key, "ExtraAttrs."), value))
+				break
+			}
+			if strings.HasPrefix(key, "extra_attrs.") {
+				qs = qs.FilterRaw("id", fmt.Sprintf("in (select id from task where extra_attrs->>'%s'='%s')", strings.TrimPrefix(key, "extra_attrs."), value))
+				break
+			}
+		}
+	}
+
+	return qs, nil
+}

src/pkg/task/dao/task_test.go

@@ -53,7 +53,7 @@ func (t *taskDAOTestSuite) SetupTest() {
 		ExecutionID: t.executionID,
 		Status:      "success",
 		StatusCode:  1,
-		ExtraAttrs:  "{}",
+		ExtraAttrs:  `{"key":"value"}`,
 	})
 	t.Require().Nil(err)
 	t.taskID = id
@@ -71,21 +71,41 @@ func (t *taskDAOTestSuite) TestCount() {
 	count, err := t.taskDAO.Count(t.ctx, &q.Query{
 		Keywords: map[string]interface{}{
 			"ExecutionID":    t.executionID,
+			"ExtraAttrs.key": "value",
 		},
 	})
 	t.Require().Nil(err)
 	t.Equal(int64(1), count)
+
+	count, err = t.taskDAO.Count(t.ctx, &q.Query{
+		Keywords: map[string]interface{}{
+			"ExecutionID":    t.executionID,
+			"ExtraAttrs.key": "incorrect-value",
+		},
+	})
+	t.Require().Nil(err)
+	t.Equal(int64(0), count)
 }
 
 func (t *taskDAOTestSuite) TestList() {
 	tasks, err := t.taskDAO.List(t.ctx, &q.Query{
 		Keywords: map[string]interface{}{
 			"ExecutionID":    t.executionID,
+			"ExtraAttrs.key": "value",
 		},
 	})
 	t.Require().Nil(err)
 	t.Require().Len(tasks, 1)
 	t.Equal(t.taskID, tasks[0].ID)
+
+	tasks, err = t.taskDAO.List(t.ctx, &q.Query{
+		Keywords: map[string]interface{}{
+			"ExecutionID":    t.executionID,
+			"ExtraAttrs.key": "incorrect-value",
+		},
+	})
+	t.Require().Nil(err)
+	t.Require().Len(tasks, 0)
 }
 
 func (t *taskDAOTestSuite) TestGet() {
@@ -140,7 +160,7 @@ func (t *taskDAOTestSuite) TestUpdateStatus() {
 
 	task, err := t.taskDAO.Get(t.ctx, t.taskID)
 	t.Require().Nil(err)
-	t.Equal(1, task.RunCount)
+	t.Equal(int32(1), task.RunCount)
 	t.True(time.Unix(statusRevision, 0).Equal(task.StartTime))
 	t.Equal(status, task.Status)
 	t.Equal(job.RunningStatus.Code(), task.StatusCode)
@@ -155,7 +175,7 @@ func (t *taskDAOTestSuite) TestUpdateStatus() {
 
 	task, err = t.taskDAO.Get(t.ctx, t.taskID)
 	t.Require().Nil(err)
-	t.Equal(1, task.RunCount)
+	t.Equal(int32(1), task.RunCount)
 	t.True(time.Unix(statusRevision, 0).Equal(task.StartTime))
 	t.Equal(status, task.Status)
 	t.Equal(job.SuccessStatus.Code(), task.StatusCode)
@@ -170,7 +190,7 @@ func (t *taskDAOTestSuite) TestUpdateStatus() {
 
 	task, err = t.taskDAO.Get(t.ctx, t.taskID)
 	t.Require().Nil(err)
-	t.Equal(2, task.RunCount)
+	t.Equal(int32(2), task.RunCount)
 	t.True(time.Unix(statusRevision, 0).Equal(task.StartTime))
 	t.Equal(status, task.Status)
 	t.Equal(job.RunningStatus.Code(), task.StatusCode)

src/pkg/task/execution.go

@@ -62,8 +62,10 @@ type ExecutionManager interface {
 	// Get the specified execution
 	Get(ctx context.Context, id int64) (execution *Execution, err error)
 	// List executions according to the query
+	// Query the "ExtraAttrs" by setting 'query.Keywords["ExtraAttrs.key"]="value"'
 	List(ctx context.Context, query *q.Query) (executions []*Execution, err error)
-	// Count counts total.
+	// Count counts total of executions according to the query.
+	// Query the "ExtraAttrs" by setting 'query.Keywords["ExtraAttrs.key"]="value"'
 	Count(ctx context.Context, query *q.Query) (int64, error)
 }
 
@@ -100,6 +102,7 @@ func (e *executionManager) Create(ctx context.Context, vendorType string, vendor
 	execution := &dao.Execution{
 		VendorType: vendorType,
 		VendorID:   vendorID,
+		Status:     job.RunningStatus.String(),
 		Trigger:    trigger,
 		ExtraAttrs: string(data),
 		StartTime:  time.Now(),
@@ -156,7 +159,11 @@ func (e *executionManager) Stop(ctx context.Context, id int64) error {
 			continue
 		}
 	}
-	return nil
+
+	// refresh the status explicitly in case that the execution status
+	// isn't refreshed by task status change hook
+	_, _, err = e.executionDAO.RefreshStatus(ctx, id)
+	return err
 }
 
 func (e *executionManager) StopAndWait(ctx context.Context, id int64, timeout time.Duration) error {

src/pkg/task/execution_test.go

@@ -104,6 +104,7 @@ func (e *executionManagerTestSuite) TestStop() {
 		},
 	}, nil)
 	e.taskMgr.On("Stop", mock.Anything, mock.Anything).Return(nil)
+	e.execDAO.On("RefreshStatus", mock.Anything, mock.Anything).Return(false, "", nil)
 	err = e.execMgr.Stop(nil, 1)
 	e.Require().Nil(err)
 	e.taskDAO.AssertExpectations(e.T())
@@ -124,6 +125,7 @@ func (e *executionManagerTestSuite) TestStopAndWait() {
 		},
 	}, nil)
 	e.taskMgr.On("Stop", mock.Anything, mock.Anything).Return(nil)
+	e.execDAO.On("RefreshStatus", mock.Anything, mock.Anything).Return(false, "", nil)
 	err := e.execMgr.StopAndWait(nil, 1, 1*time.Second)
 	e.Require().NotNil(err)
 	e.taskDAO.AssertExpectations(e.T())
@@ -145,6 +147,7 @@ func (e *executionManagerTestSuite) TestStopAndWait() {
 		},
 	}, nil)
 	e.taskMgr.On("Stop", mock.Anything, mock.Anything).Return(nil)
+	e.execDAO.On("RefreshStatus", mock.Anything, mock.Anything).Return(false, "", nil)
 	err = e.execMgr.StopAndWait(nil, 1, 1*time.Second)
 	e.Require().Nil(err)
 	e.taskDAO.AssertExpectations(e.T())

src/pkg/task/hook.go

@@ -19,9 +19,17 @@ import (
 	"fmt"
 
 	"github.com/goharbor/harbor/src/jobservice/job"
+	"github.com/goharbor/harbor/src/lib/errors"
+	"github.com/goharbor/harbor/src/lib/log"
+	"github.com/goharbor/harbor/src/lib/q"
 	"github.com/goharbor/harbor/src/pkg/task/dao"
 )
 
+var (
+	// HkHandler is a global instance of the HookHandler
+	HkHandler = NewHookHandler()
+)
+
 // NewHookHandler creates a hook handler instance
 func NewHookHandler() *HookHandler {
 	return &HookHandler{
@@ -37,31 +45,68 @@ type HookHandler struct {
 }
 
 // Handle the job status changing webhook
-func (h *HookHandler) Handle(ctx context.Context, taskID int64, sc *job.StatusChange) error {
+func (h *HookHandler) Handle(ctx context.Context, sc *job.StatusChange) error {
-	task, err := h.taskDAO.Get(ctx, taskID)
+	logger := log.GetLogger(ctx)
+
+	jobID := sc.JobID
+	// when a "KindScheduled" job is scheduled by a periodical job, it's "JobID" field
+	// is set as "87bbdee19bed5ce09c48a149@1605104520" which contains "@". In this case,
+	// read the parent periodical job ID from "sc.Metadata.UpstreamJobID"
+	if sc.Metadata.JobKind == job.KindScheduled {
+		if len(sc.Metadata.UpstreamJobID) > 0 {
+			jobID = sc.Metadata.UpstreamJobID
+		}
+	}
+	tasks, err := h.taskDAO.List(ctx, &q.Query{
+		Keywords: map[string]interface{}{
+			"JobID": jobID,
+		},
+	})
 	if err != nil {
 		return err
 	}
-	// process check in data
+	if len(tasks) == 0 {
-	if len(sc.CheckIn) > 0 {
+		return errors.New(nil).WithCode(errors.NotFoundCode).
+			WithMessage("task with job ID %s not found", sc.JobID)
+	}
+	task := tasks[0]
 	execution, err := h.executionDAO.Get(ctx, task.ExecutionID)
 	if err != nil {
 		return err
 	}
 
-		processor, exist := registry[execution.VendorType]
+	// process check in data
+	if len(sc.CheckIn) > 0 {
+		processor, exist := checkInProcessorRegistry[execution.VendorType]
 		if !exist {
-			return fmt.Errorf("the check in processor for task %d not found", taskID)
+			return fmt.Errorf("the check in processor for task %d not found", task.ID)
 		}
 		t := &Task{}
 		t.From(task)
-		return processor(ctx, t, sc)
+		return processor(ctx, t, sc.CheckIn)
 	}
 
 	// update task status
-	if err = h.taskDAO.UpdateStatus(ctx, taskID, sc.Status, sc.Metadata.Revision); err != nil {
+	if err = h.taskDAO.UpdateStatus(ctx, task.ID, sc.Status, sc.Metadata.Revision); err != nil {
 		return err
 	}
-	// update execution status
+	// run the status change post function
-	return h.executionDAO.RefreshStatus(ctx, task.ExecutionID)
+	if fc, exist := statusChangePostFuncRegistry[execution.VendorType]; exist {
+		if err = fc(ctx, task.ID, sc.Status); err != nil {
+			logger.Errorf("failed to run the task status change post function for task %d: %v", task.ID, err)
+		}
+	}
+
+	// update execution status
+	statusChanged, currentStatus, err := h.executionDAO.RefreshStatus(ctx, task.ExecutionID)
+	if err != nil {
+		return err
+	}
+	// run the status change post function
+	if fc, exist := executionStatusChangePostFuncRegistry[execution.VendorType]; exist && statusChanged {
+		if err = fc(ctx, task.ExecutionID, currentStatus); err != nil {
+			logger.Errorf("failed to run the execution status change post function for execution %d: %v", task.ExecutionID, err)
+		}
+	}
+	return nil
 }

src/pkg/task/hook_test.go

@@ -43,10 +43,13 @@ func (h *hookHandlerTestSuite) SetupTest() {
 
 func (h *hookHandlerTestSuite) TestHandle() {
 	// handle check in data
-	registry["test"] = func(ctx context.Context, task *Task, change *job.StatusChange) (err error) { return nil }
+	checkInProcessorRegistry["test"] = func(ctx context.Context, task *Task, data string) (err error) { return nil }
-	h.taskDAO.On("Get", mock.Anything, mock.Anything).Return(&dao.Task{
+	defer delete(checkInProcessorRegistry, "test")
+	h.taskDAO.On("List", mock.Anything, mock.Anything).Return([]*dao.Task{
+		{
 			ID:          1,
 			ExecutionID: 1,
+		},
 	}, nil)
 	h.execDAO.On("Get", mock.Anything, mock.Anything).Return(&dao.Execution{
 		ID:         1,
@@ -54,8 +57,9 @@ func (h *hookHandlerTestSuite) TestHandle() {
 	}, nil)
 	sc := &job.StatusChange{
 		CheckIn:  "data",
+		Metadata: &job.StatsInfo{},
 	}
-	err := h.handler.Handle(nil, 1, sc)
+	err := h.handler.Handle(nil, sc)
 	h.Require().Nil(err)
 	h.taskDAO.AssertExpectations(h.T())
 	h.execDAO.AssertExpectations(h.T())
@@ -64,21 +68,28 @@ func (h *hookHandlerTestSuite) TestHandle() {
 	h.SetupTest()
 
 	// handle status changing
-	h.taskDAO.On("Get", mock.Anything, mock.Anything).Return(&dao.Task{
+	h.taskDAO.On("List", mock.Anything, mock.Anything).Return([]*dao.Task{
+		{
 			ID:          1,
 			ExecutionID: 1,
+		},
 	}, nil)
 	h.taskDAO.On("UpdateStatus", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil)
-	h.execDAO.On("RefreshStatus", mock.Anything, mock.Anything).Return(nil)
+	h.execDAO.On("Get", mock.Anything, mock.Anything).Return(&dao.Execution{
+		ID:         1,
+		VendorType: "test",
+	}, nil)
+	h.execDAO.On("RefreshStatus", mock.Anything, mock.Anything).Return(true, job.RunningStatus.String(), nil)
 	sc = &job.StatusChange{
 		Status: job.SuccessStatus.String(),
 		Metadata: &job.StatsInfo{
 			Revision: time.Now().Unix(),
 		},
 	}
-	err = h.handler.Handle(nil, 1, sc)
+	err = h.handler.Handle(nil, sc)
 	h.Require().Nil(err)
 	h.taskDAO.AssertExpectations(h.T())
+	h.execDAO.AssertExpectations(h.T())
 }
 
 func TestHookHandlerTestSuite(t *testing.T) {

src/pkg/task/mock_execution_dao_test.go

@@ -142,17 +142,31 @@ func (_m *mockExecutionDAO) List(ctx context.Context, query *q.Query) ([]*dao.Ex
 }
 
 // RefreshStatus provides a mock function with given fields: ctx, id
-func (_m *mockExecutionDAO) RefreshStatus(ctx context.Context, id int64) error {
+func (_m *mockExecutionDAO) RefreshStatus(ctx context.Context, id int64) (bool, string, error) {
 	ret := _m.Called(ctx, id)
 
-	var r0 error
+	var r0 bool
-	if rf, ok := ret.Get(0).(func(context.Context, int64) error); ok {
+	if rf, ok := ret.Get(0).(func(context.Context, int64) bool); ok {
 		r0 = rf(ctx, id)
 	} else {
-		r0 = ret.Error(0)
+		r0 = ret.Get(0).(bool)
 	}
 
-	return r0
+	var r1 string
+	if rf, ok := ret.Get(1).(func(context.Context, int64) string); ok {
+		r1 = rf(ctx, id)
+	} else {
+		r1 = ret.Get(1).(string)
+	}
+
+	var r2 error
+	if rf, ok := ret.Get(2).(func(context.Context, int64) error); ok {
+		r2 = rf(ctx, id)
+	} else {
+		r2 = ret.Error(2)
+	}
+
+	return r0, r1, r2
 }
 
 // Update provides a mock function with given fields: ctx, execution, props

src/pkg/task/model.go

@@ -25,13 +25,6 @@ import (
 
 // const definitions
 const (
-	ExecutionVendorTypeReplication       = "REPLICATION"
-	ExecutionVendorTypeGarbageCollection = "GARBAGE_COLLECTION"
-	ExecutionVendorTypeRetention         = "RETENTION"
-	ExecutionVendorTypeScan              = "SCAN"
-	ExecutionVendorTypeScanAll           = "SCAN_ALL"
-	ExecutionVendorTypeScheduler         = "SCHEDULER"
-
 	ExecutionTriggerManual   = "MANUAL"
 	ExecutionTriggerSchedule = "SCHEDULE"
 	ExecutionTriggerEvent    = "EVENT"
@@ -62,13 +55,17 @@ type Execution struct {
 // Task is the unit for running. It stores the jobservice job records and related information
 type Task struct {
 	ID int64 `json:"id"`
+	// indicate the task type: replication/GC/retention/scan/etc.
+	VendorType  string `json:"vendor_type"`
 	ExecutionID int64  `json:"execution_id"`
 	Status      string `json:"status"`
 	// the detail message to explain the status in some cases. e.g.
 	// When the job is failed to submit to jobservice, this field can be used to explain the reason
 	StatusMessage string `json:"status_message"`
 	// the underlying job may retry several times
-	RunCount int `json:"run_count"`
+	RunCount int32 `json:"run_count"`
+	// the ID of jobservice job
+	JobID string `json:"job_id"`
 	// the customized attributes for different kinds of consumers
 	ExtraAttrs map[string]interface{} `json:"extra_attrs"`
 	// the time that the task record created
@@ -82,10 +79,12 @@ type Task struct {
 // From constructs a task from DAO model
 func (t *Task) From(task *dao.Task) {
 	t.ID = task.ID
+	t.VendorType = task.VendorType
 	t.ExecutionID = task.ExecutionID
 	t.Status = task.Status
 	t.StatusMessage = task.StatusMessage
 	t.RunCount = task.RunCount
+	t.JobID = task.JobID
 	t.CreationTime = task.CreationTime
 	t.StartTime = task.StartTime
 	t.UpdateTime = task.UpdateTime
@@ -100,6 +99,22 @@ func (t *Task) From(task *dao.Task) {
 	}
 }
 
+// GetStringFromExtraAttrs returns the string value specified by key
+func (t *Task) GetStringFromExtraAttrs(key string) string {
+	if len(t.ExtraAttrs) == 0 {
+		return ""
+	}
+	rt, exist := t.ExtraAttrs[key]
+	if !exist {
+		return ""
+	}
+	str, ok := rt.(string)
+	if !ok {
+		return ""
+	}
+	return str
+}
+
 // Job is the model represents the requested jobservice job
 type Job struct {
 	Name       string

src/pkg/task/registry.go

@@ -0,0 +1,62 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package task
+
+import (
+	"context"
+	"fmt"
+)
+
+var (
+	checkInProcessorRegistry              = map[string]CheckInProcessor{}
+	statusChangePostFuncRegistry          = map[string]StatusChangePostFunc{}
+	executionStatusChangePostFuncRegistry = map[string]ExecutionStatusChangePostFunc{}
+)
+
+// CheckInProcessor is the processor to process the check in data which is sent by jobservice via webhook
+type CheckInProcessor func(ctx context.Context, task *Task, data string) (err error)
+
+// StatusChangePostFunc is the function called after the task status changed
+type StatusChangePostFunc func(ctx context.Context, taskID int64, status string) (err error)
+
+// ExecutionStatusChangePostFunc is the function called after the execution status changed
+type ExecutionStatusChangePostFunc func(ctx context.Context, executionID int64, status string) (err error)
+
+// RegisterCheckInProcessor registers check in processor for the specific vendor type
+func RegisterCheckInProcessor(vendorType string, processor CheckInProcessor) error {
+	if _, exist := checkInProcessorRegistry[vendorType]; exist {
+		return fmt.Errorf("check in processor for %s already exists", vendorType)
+	}
+	checkInProcessorRegistry[vendorType] = processor
+	return nil
+}
+
+// RegisterTaskStatusChangePostFunc registers a task status change post function for the specific vendor type
+func RegisterTaskStatusChangePostFunc(vendorType string, fc StatusChangePostFunc) error {
+	if _, exist := statusChangePostFuncRegistry[vendorType]; exist {
+		return fmt.Errorf("the task status change post function for %s already exists", vendorType)
+	}
+	statusChangePostFuncRegistry[vendorType] = fc
+	return nil
+}
+
+// RegisterExecutionStatusChangePostFunc registers an execution status change post function for the specific vendor type
+func RegisterExecutionStatusChangePostFunc(vendorType string, fc ExecutionStatusChangePostFunc) error {
+	if _, exist := executionStatusChangePostFuncRegistry[vendorType]; exist {
+		return fmt.Errorf("the execution status change post function for %s already exists", vendorType)
+	}
+	executionStatusChangePostFuncRegistry[vendorType] = fc
+	return nil
+}

src/pkg/task/registry_test.go

@@ -0,0 +1,48 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package task
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRegisterCheckInProcessor(t *testing.T) {
+	err := RegisterCheckInProcessor("test", nil)
+	assert.Nil(t, err)
+
+	// already exist
+	err = RegisterCheckInProcessor("test", nil)
+	assert.NotNil(t, err)
+}
+
+func TestRegisterTaskStatusChangePostFunc(t *testing.T) {
+	err := RegisterTaskStatusChangePostFunc("test", nil)
+	assert.Nil(t, err)
+
+	// already exist
+	err = RegisterTaskStatusChangePostFunc("test", nil)
+	assert.NotNil(t, err)
+}
+
+func TestRegisterExecutionStatusChangePostFunc(t *testing.T) {
+	err := RegisterExecutionStatusChangePostFunc("test", nil)
+	assert.Nil(t, err)
+
+	// already exist
+	err = RegisterExecutionStatusChangePostFunc("test", nil)
+	assert.NotNil(t, err)
+}

src/pkg/task/task.go

@@ -47,10 +47,12 @@ type Manager interface {
 	// Get the specified task
 	Get(ctx context.Context, id int64) (task *Task, err error)
 	// List the tasks according to the query
+	// Query the "ExtraAttrs" by setting 'query.Keywords["ExtraAttrs.key"]="value"'
 	List(ctx context.Context, query *q.Query) (tasks []*Task, err error)
 	// Get the log of the specified task
 	GetLog(ctx context.Context, id int64) (log []byte, err error)
-	// Count counts total.
+	// Count counts total of tasks according to the query.
+	// Query the "ExtraAttrs" by setting 'query.Keywords["ExtraAttrs.key"]="value"'
 	Count(ctx context.Context, query *q.Query) (int64, error)
 }
 
@@ -58,6 +60,7 @@ type Manager interface {
 func NewManager() Manager {
 	return &manager{
 		dao:      dao.NewTaskDAO(),
+		execDAO:  dao.NewExecutionDAO(),
 		jsClient: cjob.GlobalClient,
 		coreURL:  config.GetCoreURL(),
 	}
@@ -65,6 +68,7 @@ func NewManager() Manager {
 
 type manager struct {
 	dao      dao.TaskDAO
+	execDAO  dao.ExecutionDAO
 	jsClient cjob.Client
 	coreURL  string
 }
@@ -84,22 +88,11 @@ func (m *manager) Create(ctx context.Context, executionID int64, jb *Job, extraA
 	// submit job to jobservice
 	jobID, err := m.submitJob(ctx, id, jb)
 	if err != nil {
-		// failed to submit job to jobservice, update the status of task to error
+		// failed to submit job to jobservice, delete the task record
-		err = fmt.Errorf("failed to submit job to jobservice: %v", err)
+		if err := m.dao.Delete(ctx, id); err != nil {
-		log.Error(err)
+			log.Errorf("failed to delete the task %d: %v", id, err)
-		now := time.Now()
-		err = m.dao.Update(ctx, &dao.Task{
-			ID:            id,
-			Status:        job.ErrorStatus.String(),
-			StatusCode:    job.ErrorStatus.Code(),
-			StatusMessage: err.Error(),
-			UpdateTime:    now,
-			EndTime:       now,
-		}, "Status", "StatusCode", "StatusMessage", "UpdateTime", "EndTime")
-		if err != nil {
-			log.Errorf("failed to update task %d: %v", id, err)
 		}
-		return id, nil
+		return 0, err
 	}
 
 	log.Debugf("the task %d is submitted to jobservice, the job ID is %s", id, jobID)
@@ -116,6 +109,10 @@ func (m *manager) Create(ctx context.Context, executionID int64, jb *Job, extraA
 }
 
 func (m *manager) createTaskRecord(ctx context.Context, executionID int64, extraAttrs ...map[string]interface{}) (int64, error) {
+	exec, err := m.execDAO.Get(ctx, executionID)
+	if err != nil {
+		return 0, err
+	}
 	extras := map[string]interface{}{}
 	if len(extraAttrs) > 0 && extraAttrs[0] != nil {
 		extras = extraAttrs[0]
@@ -127,6 +124,7 @@ func (m *manager) createTaskRecord(ctx context.Context, executionID int64, extra
 
 	now := time.Now()
 	return m.dao.Create(ctx, &dao.Task{
+		VendorType:   exec.VendorType,
 		ExecutionID:  executionID,
 		Status:       job.PendingStatus.String(),
 		StatusCode:   job.PendingStatus.Code(),

src/pkg/task/task_test.go

@@ -31,14 +31,17 @@ type taskManagerTestSuite struct {
 	suite.Suite
 	mgr      *manager
 	dao      *mockTaskDAO
+	execDAO  *mockExecutionDAO
 	jsClient *mockJobserviceClient
 }
 
 func (t *taskManagerTestSuite) SetupTest() {
 	t.dao = &mockTaskDAO{}
+	t.execDAO = &mockExecutionDAO{}
 	t.jsClient = &mockJobserviceClient{}
 	t.mgr = &manager{
 		dao:      t.dao,
+		execDAO:  t.execDAO,
 		jsClient: t.jsClient,
 	}
 }
@@ -53,6 +56,7 @@ func (t *taskManagerTestSuite) TestCount() {
 
 func (t *taskManagerTestSuite) TestCreate() {
 	// success to submit job to jobservice
+	t.execDAO.On("Get", mock.Anything, mock.Anything).Return(&dao.Execution{}, nil)
 	t.dao.On("Create", mock.Anything, mock.Anything).Return(int64(1), nil)
 	t.jsClient.On("SubmitJob", mock.Anything).Return("1", nil)
 	t.dao.On("Update", mock.Anything, mock.Anything, mock.Anything).Return(nil)
@@ -61,21 +65,22 @@ func (t *taskManagerTestSuite) TestCreate() {
 	t.Require().Nil(err)
 	t.Equal(int64(1), id)
 	t.dao.AssertExpectations(t.T())
+	t.execDAO.AssertExpectations(t.T())
 	t.jsClient.AssertExpectations(t.T())
 
 	// reset mock
 	t.SetupTest()
 
 	// failed to submit job to jobservice
+	t.execDAO.On("Get", mock.Anything, mock.Anything).Return(&dao.Execution{}, nil)
 	t.dao.On("Create", mock.Anything, mock.Anything).Return(int64(1), nil)
 	t.jsClient.On("SubmitJob", mock.Anything).Return("", errors.New("error"))
-	t.dao.On("Update", mock.Anything, mock.Anything, mock.Anything,
+	t.dao.On("Delete", mock.Anything, mock.Anything).Return(nil)
-		mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil)
 
 	id, err = t.mgr.Create(nil, 1, &Job{}, map[string]interface{}{"a": "b"})
-	t.Require().Nil(err)
+	t.Require().NotNil(err)
-	t.Equal(int64(1), id)
 	t.dao.AssertExpectations(t.T())
+	t.execDAO.AssertExpectations(t.T())
 	t.jsClient.AssertExpectations(t.T())
 }
 

src/replication/config/config.go

@@ -23,9 +23,7 @@ var (
 type Configuration struct {
 	CoreURL         string
 	TokenServiceURL string
-	JobserviceURL   string
 	SecretKey       string
 	// TODO consider to use a specified secret for replication
-	CoreSecret       string
 	JobserviceSecret string
 }

src/replication/dao/base.go

@@ -1,13 +0,0 @@
-package dao
-
-import "github.com/astaxie/beego/orm"
-
-func paginateForQuerySetter(qs orm.QuerySeter, page, size int64) orm.QuerySeter {
-	if size > 0 {
-		qs = qs.Limit(size)
-		if page > 0 {
-			qs = qs.Offset((page - 1) * size)
-		}
-	}
-	return qs
-}

src/replication/dao/execution.go

@@ -1,355 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package dao
-
-import (
-	"fmt"
-	"time"
-
-	"github.com/astaxie/beego/orm"
-	"github.com/goharbor/harbor/src/common/dao"
-	"github.com/goharbor/harbor/src/lib/log"
-	"github.com/goharbor/harbor/src/replication/dao/models"
-)
-
-// AddExecution ...
-func AddExecution(execution *models.Execution) (int64, error) {
-	o := dao.GetOrmer()
-	now := time.Now()
-	execution.StartTime = now
-
-	return o.Insert(execution)
-}
-
-// GetTotalOfExecutions returns the total count of replication execution
-func GetTotalOfExecutions(query ...*models.ExecutionQuery) (int64, error) {
-	qs := executionQueryConditions(query...)
-	return qs.Count()
-}
-
-// GetExecutions ...
-func GetExecutions(query ...*models.ExecutionQuery) ([]*models.Execution, error) {
-	executions := []*models.Execution{}
-
-	qs := executionQueryConditions(query...)
-	if len(query) > 0 && query[0] != nil {
-		qs = paginateForQuerySetter(qs, query[0].Page, query[0].Size)
-	}
-
-	qs = qs.OrderBy("-StartTime")
-
-	_, err := qs.All(&executions)
-	if err != nil || len(executions) == 0 {
-		return executions, err
-	}
-	for _, e := range executions {
-		fillExecution(e)
-	}
-	return executions, err
-}
-
-func executionQueryConditions(query ...*models.ExecutionQuery) orm.QuerySeter {
-	qs := dao.GetOrmer().QueryTable(new(models.Execution))
-	if len(query) == 0 || query[0] == nil {
-		return qs
-	}
-
-	q := query[0]
-	if q.PolicyID != 0 {
-		qs = qs.Filter("PolicyID", q.PolicyID)
-	}
-	if len(q.Trigger) > 0 {
-		qs = qs.Filter("Trigger", q.Trigger)
-	}
-	if len(q.Statuses) > 0 {
-		qs = qs.Filter("Status__in", q.Statuses)
-	}
-	return qs
-}
-
-// GetExecution ...
-func GetExecution(id int64) (*models.Execution, error) {
-	o := dao.GetOrmer()
-	t := models.Execution{ID: id}
-	err := o.Read(&t)
-	if err == orm.ErrNoRows {
-		return nil, nil
-	}
-	if err != nil {
-		return nil, err
-	}
-	fillExecution(&t)
-	return &t, err
-}
-
-// fillExecution will fill the statistics data and status by tasks data
-func fillExecution(execution *models.Execution) error {
-	if executionFinished(execution.Status) {
-		return nil
-	}
-
-	o := dao.GetOrmer()
-	sql := `select status, count(*) as c from replication_task where execution_id = ? group by status`
-	queryParam := make([]interface{}, 1)
-	queryParam = append(queryParam, execution.ID)
-
-	dt := []*models.TaskStat{}
-	count, err := o.Raw(sql, queryParam).QueryRows(&dt)
-
-	if err != nil {
-		log.Errorf("Query tasks error execution %d: %v", execution.ID, err)
-		return err
-	}
-
-	if count == 0 {
-		return nil
-	}
-
-	total := 0
-	for _, d := range dt {
-		status, _ := getStatus(d.Status)
-		updateStatusCount(execution, status, d.C)
-		total += d.C
-	}
-
-	if execution.Total != total {
-		log.Debugf("execution task count inconsistent and fixed, executionID=%d, execution.total=%d, tasks.count=%d",
-			execution.ID, execution.Total, total)
-		execution.Total = total
-	}
-	resetExecutionStatus(execution)
-
-	return nil
-}
-
-func getStatus(status string) (string, error) {
-	switch status {
-	case models.TaskStatusInitialized, models.TaskStatusPending, models.TaskStatusInProgress:
-		return models.ExecutionStatusInProgress, nil
-	case models.TaskStatusSucceed:
-		return models.ExecutionStatusSucceed, nil
-	case models.TaskStatusStopped:
-		return models.ExecutionStatusStopped, nil
-	case models.TaskStatusFailed:
-		return models.ExecutionStatusFailed, nil
-	}
-	return "", fmt.Errorf("Not support task status ")
-}
-
-func updateStatusCount(execution *models.Execution, status string, delta int) error {
-	switch status {
-	case models.ExecutionStatusInProgress:
-		execution.InProgress += delta
-	case models.ExecutionStatusSucceed:
-		execution.Succeed += delta
-	case models.ExecutionStatusStopped:
-		execution.Stopped += delta
-	case models.ExecutionStatusFailed:
-		execution.Failed += delta
-	}
-	return nil
-}
-
-func resetExecutionStatus(execution *models.Execution) error {
-	execution.Status = generateStatus(execution)
-	if executionFinished(execution.Status) {
-		o := dao.GetOrmer()
-		sql := `select max(end_time) from replication_task where execution_id = ?`
-		queryParam := make([]interface{}, 1)
-		queryParam = append(queryParam, execution.ID)
-
-		var et time.Time
-		err := o.Raw(sql, queryParam).QueryRow(&et)
-		if err != nil {
-			log.Errorf("Query end_time from tasks error execution %d: %v", execution.ID, err)
-			et = time.Now()
-		}
-		execution.EndTime = et
-	}
-	return nil
-}
-
-func generateStatus(execution *models.Execution) string {
-	if execution.InProgress > 0 {
-		return models.ExecutionStatusInProgress
-	} else if execution.Failed > 0 {
-		return models.ExecutionStatusFailed
-	} else if execution.Stopped > 0 {
-		return models.ExecutionStatusStopped
-	}
-	return models.ExecutionStatusSucceed
-}
-
-func executionFinished(status string) bool {
-	if status == models.ExecutionStatusStopped ||
-		status == models.ExecutionStatusSucceed ||
-		status == models.ExecutionStatusFailed {
-		return true
-	}
-	return false
-}
-
-// DeleteExecution ...
-func DeleteExecution(id int64) error {
-	o := dao.GetOrmer()
-	_, err := o.Delete(&models.Execution{ID: id})
-	return err
-}
-
-// DeleteAllExecutions ...
-func DeleteAllExecutions(policyID int64) error {
-	o := dao.GetOrmer()
-	_, err := o.Delete(&models.Execution{PolicyID: policyID}, "PolicyID")
-	return err
-}
-
-// UpdateExecution ...
-func UpdateExecution(execution *models.Execution, props ...string) (int64, error) {
-	if execution.ID == 0 {
-		return 0, fmt.Errorf("execution ID is empty")
-	}
-	o := dao.GetOrmer()
-	return o.Update(execution, props...)
-}
-
-// AddTask ...
-func AddTask(task *models.Task) (int64, error) {
-	o := dao.GetOrmer()
-	now := time.Now()
-	task.StartTime = now
-
-	return o.Insert(task)
-}
-
-// GetTask ...
-func GetTask(id int64) (*models.Task, error) {
-	o := dao.GetOrmer()
-	sql := `select * from replication_task where id = ?`
-
-	var task models.Task
-
-	if err := o.Raw(sql, id).QueryRow(&task); err != nil {
-		if err == orm.ErrNoRows {
-			return nil, nil
-		}
-		return nil, err
-	}
-
-	return &task, nil
-}
-
-// GetTotalOfTasks ...
-func GetTotalOfTasks(query ...*models.TaskQuery) (int64, error) {
-	qs := taskQueryConditions(query...)
-	return qs.Count()
-}
-
-// GetTasks ...
-func GetTasks(query ...*models.TaskQuery) ([]*models.Task, error) {
-	tasks := []*models.Task{}
-
-	qs := taskQueryConditions(query...)
-	if len(query) > 0 && query[0] != nil {
-		qs = paginateForQuerySetter(qs, query[0].Page, query[0].Size)
-	}
-
-	qs = qs.OrderBy("-StartTime")
-
-	_, err := qs.All(&tasks)
-	return tasks, err
-}
-
-func taskQueryConditions(query ...*models.TaskQuery) orm.QuerySeter {
-	qs := dao.GetOrmer().QueryTable(new(models.Task))
-	if len(query) == 0 || query[0] == nil {
-		return qs
-	}
-
-	q := query[0]
-	if q.ExecutionID != 0 {
-		qs = qs.Filter("ExecutionID", q.ExecutionID)
-	}
-	if len(q.JobID) > 0 {
-		qs = qs.Filter("JobID", q.JobID)
-	}
-	if len(q.ResourceType) > 0 {
-		qs = qs.Filter("ResourceType", q.ResourceType)
-	}
-	if len(q.Statuses) > 0 {
-		qs = qs.Filter("Status__in", q.Statuses)
-	}
-	return qs
-}
-
-// DeleteTask ...
-func DeleteTask(id int64) error {
-	o := dao.GetOrmer()
-	_, err := o.Delete(&models.Task{ID: id})
-	return err
-}
-
-// DeleteAllTasks ...
-func DeleteAllTasks(executionID int64) error {
-	o := dao.GetOrmer()
-	_, err := o.Delete(&models.Task{ExecutionID: executionID}, "ExecutionID")
-	return err
-}
-
-// UpdateTask ...
-func UpdateTask(task *models.Task, props ...string) (int64, error) {
-	if task.ID == 0 {
-		return 0, fmt.Errorf("task ID is empty")
-	}
-	o := dao.GetOrmer()
-	return o.Update(task, props...)
-}
-
-// UpdateTaskStatus updates the status of task.
-// The implementation uses raw sql rather than QuerySetter.Filter... as QuerySetter
-// will generate sql like:
-//   `UPDATE "replication_task" SET "end_time" = $1, "status" = $2
-//     WHERE "id" IN ( SELECT T0."id" FROM "replication_task" T0 WHERE T0."id" = $3
-//     AND T0."status" IN ($4, $5, $6))]`
-// which is not a "single" sql statement, this will cause issues when running in concurrency
-func UpdateTaskStatus(id int64, status string, statusRevision int64, statusCondition ...string) (int64, error) {
-	params := []interface{}{}
-	sql := `update replication_task set status = ?, status_revision = ?, end_time = ? `
-	params = append(params, status, statusRevision)
-	var t time.Time
-	// when the task is in final status, update the endtime
-	// when the task re-runs again, the endtime should be cleared
-	// so set the endtime to null if the task isn't in final status
-	if taskFinished(status) {
-		t = time.Now()
-	}
-	params = append(params, t)
-
-	sql += fmt.Sprintf(`where id = ? and (status_revision < ? or status_revision = ? and status in (%s)) `, dao.ParamPlaceholderForIn(len(statusCondition)))
-	params = append(params, id, statusRevision, statusRevision, statusCondition)
-
-	result, err := dao.GetOrmer().Raw(sql, params...).Exec()
-	if err != nil {
-		return 0, err
-	}
-	n, _ := result.RowsAffected()
-	if n > 0 {
-		log.Debugf("update task status %d: -> %s", id, status)
-	}
-	return n, err
-}
-
-func taskFinished(status string) bool {
-	return status == models.TaskStatusFailed || status == models.TaskStatusStopped || status == models.TaskStatusSucceed
-}

src/replication/dao/execution_test.go

@@ -1,294 +0,0 @@
-package dao
-
-import (
-	"testing"
-	"time"
-
-	"github.com/goharbor/harbor/src/replication/dao/models"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestMethodOfExecution(t *testing.T) {
-	execution1 := &models.Execution{
-		PolicyID:   11209,
-		Status:     "InProgress",
-		StatusText: "None",
-		Total:      12,
-		Failed:     0,
-		Succeed:    7,
-		InProgress: 5,
-		Stopped:    0,
-		Trigger:    "Event",
-		StartTime:  time.Now(),
-	}
-	execution2 := &models.Execution{
-		PolicyID:   11209,
-		Status:     "Failed",
-		StatusText: "Network error",
-		Total:      9,
-		Failed:     1,
-		Succeed:    8,
-		InProgress: 0,
-		Stopped:    0,
-		Trigger:    "Manual",
-		StartTime:  time.Now(),
-	}
-
-	// test add
-	id1, err := AddExecution(execution1)
-	require.Nil(t, err)
-
-	_, err = AddExecution(execution2)
-	require.Nil(t, err)
-
-	// test list
-	query := &models.ExecutionQuery{
-		Statuses: []string{"InProgress", "Failed"},
-		Pagination: models.Pagination{
-			Page: 1,
-			Size: 10,
-		},
-	}
-	executions, err := GetExecutions(query)
-	require.Nil(t, err)
-	assert.Equal(t, 2, len(executions))
-
-	total, err := GetTotalOfExecutions(query)
-	require.Nil(t, err)
-	assert.Equal(t, int64(2), total)
-
-	// test get
-	execution, err := GetExecution(id1)
-	require.Nil(t, err)
-	assert.Equal(t, execution1.Status, execution.Status)
-
-	// test update
-	executionNew := &models.Execution{
-		ID:         id1,
-		Status:     "Succeed",
-		Succeed:    12,
-		InProgress: 0,
-		EndTime:    time.Now(),
-	}
-	n, err := UpdateExecution(executionNew, models.ExecutionPropsName.Status, models.ExecutionPropsName.Succeed, models.ExecutionPropsName.InProgress,
-		models.ExecutionPropsName.EndTime)
-	require.Nil(t, err)
-	assert.Equal(t, int64(1), n)
-
-	// test delete
-	require.Nil(t, DeleteExecution(execution1.ID))
-	execution, err = GetExecution(execution1.ID)
-	require.Nil(t, err)
-	require.Nil(t, execution)
-
-	// test delete all
-	require.Nil(t, DeleteAllExecutions(execution1.PolicyID))
-	query = &models.ExecutionQuery{}
-	n, err = GetTotalOfExecutions(query)
-	require.Nil(t, err)
-	assert.Equal(t, int64(0), n)
-}
-
-func TestMethodOfTask(t *testing.T) {
-	now := time.Now()
-	task1 := &models.Task{
-		ExecutionID:    112200,
-		ResourceType:   "resourceType1",
-		SrcResource:    "srcResource1",
-		DstResource:    "dstResource1",
-		JobID:          "jobID1",
-		Status:         "Initialized",
-		StatusRevision: 1,
-		StartTime:      now,
-	}
-	task2 := &models.Task{
-		ExecutionID:    112200,
-		ResourceType:   "resourceType2",
-		SrcResource:    "srcResource2",
-		DstResource:    "dstResource2",
-		JobID:          "jobID2",
-		Status:         "Stopped",
-		StatusRevision: 1,
-		StartTime:      now,
-		EndTime:        now,
-	}
-
-	// test add
-	id1, err := AddTask(task1)
-	require.Nil(t, err)
-
-	_, err = AddTask(task2)
-	require.Nil(t, err)
-
-	// test list
-	query := &models.TaskQuery{
-		ResourceType: "resourceType1",
-		Pagination: models.Pagination{
-			Page: 1,
-			Size: 10,
-		},
-	}
-	tasks, err := GetTasks(query)
-	require.Nil(t, err)
-	assert.Equal(t, 1, len(tasks))
-
-	total, err := GetTotalOfTasks(query)
-	require.Nil(t, err)
-	assert.Equal(t, int64(1), total)
-
-	// test get
-	task, err := GetTask(id1)
-	require.Nil(t, err)
-	assert.Equal(t, task1.Status, task.Status)
-
-	// test update
-	taskNew := &models.Task{
-		ID:      id1,
-		Status:  "Failed",
-		EndTime: now,
-	}
-	n, err := UpdateTask(taskNew, models.TaskPropsName.Status, models.TaskPropsName.EndTime)
-	require.Nil(t, err)
-	assert.Equal(t, int64(1), n)
-
-	// test update status
-	n, err = UpdateTaskStatus(id1, "Succeed", 2, "Initialized")
-	require.Nil(t, err)
-	assert.Equal(t, int64(1), n)
-	task, _ = GetTask(id1)
-	assert.Equal(t, "Succeed", task.Status)
-	assert.Equal(t, int64(2), task.StatusRevision)
-
-	// test delete
-	require.Nil(t, DeleteTask(id1))
-	task, err = GetTask(id1)
-	require.Nil(t, err)
-	require.Nil(t, task)
-
-	// test delete all
-	require.Nil(t, DeleteAllTasks(task1.ExecutionID))
-	query = &models.TaskQuery{}
-	n, err = GetTotalOfTasks(query)
-	require.Nil(t, err)
-	assert.Equal(t, int64(0), n)
-}
-
-func TestExecutionFill(t *testing.T) {
-	now := time.Now()
-	execution := &models.Execution{
-		PolicyID:   11209,
-		Status:     "InProgress",
-		StatusText: "None",
-		Total:      2,
-		Trigger:    "Event",
-		StartTime:  time.Now(),
-	}
-	executionID, _ := AddExecution(execution)
-	et1, _ := time.Parse("2006-01-02 15:04:05", "2019-03-21 08:01:01")
-	et2, _ := time.Parse("2006-01-02 15:04:05", "2019-04-01 10:11:53")
-	task1 := &models.Task{
-		ID:           20191,
-		ExecutionID:  executionID,
-		ResourceType: "resourceType1",
-		SrcResource:  "srcResource1",
-		DstResource:  "dstResource1",
-		JobID:        "jobID1",
-		Status:       "Succeed",
-		StartTime:    now,
-		EndTime:      et1,
-	}
-	task2 := &models.Task{
-		ID:           20192,
-		ExecutionID:  executionID,
-		ResourceType: "resourceType2",
-		SrcResource:  "srcResource2",
-		DstResource:  "dstResource2",
-		JobID:        "jobID2",
-		Status:       "Stopped",
-		StartTime:    now,
-		EndTime:      et2,
-	}
-	AddTask(task1)
-	AddTask(task2)
-
-	defer func() {
-		DeleteAllTasks(executionID)
-		DeleteAllExecutions(11209)
-	}()
-
-	// query and fill
-	exe, err := GetExecution(executionID)
-	require.Nil(t, err)
-	assert.Equal(t, "Stopped", exe.Status)
-	assert.Equal(t, 0, exe.InProgress)
-	assert.Equal(t, 1, exe.Stopped)
-	assert.Equal(t, 0, exe.Failed)
-	assert.Equal(t, 1, exe.Succeed)
-	assert.Equal(t, et2.Second(), exe.EndTime.Second())
-}
-
-func TestExecutionFill2(t *testing.T) {
-	now := time.Now()
-	execution := &models.Execution{
-		PolicyID:   11209,
-		Status:     "InProgress",
-		StatusText: "None",
-		Total:      2,
-		Trigger:    "Event",
-		StartTime:  time.Now(),
-	}
-	executionID, _ := AddExecution(execution)
-	task1 := &models.Task{
-		ID:             20191,
-		ExecutionID:    executionID,
-		ResourceType:   "resourceType1",
-		SrcResource:    "srcResource1",
-		DstResource:    "dstResource1",
-		JobID:          "jobID1",
-		Status:         models.TaskStatusInProgress,
-		StatusRevision: 1,
-		StartTime:      now,
-	}
-	task2 := &models.Task{
-		ID:             20192,
-		ExecutionID:    executionID,
-		ResourceType:   "resourceType2",
-		SrcResource:    "srcResource2",
-		DstResource:    "dstResource2",
-		JobID:          "jobID2",
-		Status:         "Stopped",
-		StatusRevision: 1,
-		StartTime:      now,
-		EndTime:        now,
-	}
-	taskID1, _ := AddTask(task1)
-	AddTask(task2)
-
-	defer func() {
-		DeleteAllTasks(executionID)
-		DeleteAllExecutions(11209)
-	}()
-
-	// query and fill
-	exe, err := GetExecution(executionID)
-	require.Nil(t, err)
-	assert.Equal(t, models.ExecutionStatusInProgress, exe.Status)
-	assert.Equal(t, 1, exe.InProgress)
-	assert.Equal(t, 1, exe.Stopped)
-	assert.Equal(t, 0, exe.Failed)
-	assert.Equal(t, 0, exe.Succeed)
-
-	// update task status and query and fill
-	UpdateTaskStatus(taskID1, models.TaskStatusFailed, 2, models.TaskStatusInProgress)
-	exes, err := GetExecutions(&models.ExecutionQuery{
-		PolicyID: 11209,
-	})
-	require.Nil(t, err)
-	assert.Equal(t, 1, len(exes))
-	assert.Equal(t, models.ExecutionStatusFailed, exes[0].Status)
-	assert.Equal(t, 0, exes[0].InProgress)
-	assert.Equal(t, 1, exes[0].Stopped)
-	assert.Equal(t, 1, exes[0].Failed)
-	assert.Equal(t, 0, exes[0].Succeed)
-}

src/replication/dao/models/base.go

@@ -7,14 +7,5 @@ import (
 func init() {
 	orm.RegisterModel(
 		new(Registry),
-		new(RepPolicy),
+		new(RepPolicy))
-		new(Execution),
-		new(Task),
-		new(ScheduleJob))
-}
-
-// Pagination ...
-type Pagination struct {
-	Page int64
-	Size int64
 }

src/replication/dao/models/execution.go

@@ -1,156 +0,0 @@
-package models
-
-import (
-	"time"
-
-	"github.com/goharbor/harbor/src/replication/model"
-)
-
-const (
-	// ExecutionTable is the table name for replication executions
-	ExecutionTable = "replication_execution"
-	// TaskTable is table name for replication tasks
-	TaskTable = "replication_task"
-)
-
-// execution/task status/trigger const
-const (
-	ExecutionStatusFailed     string = "Failed"
-	ExecutionStatusSucceed    string = "Succeed"
-	ExecutionStatusStopped    string = "Stopped"
-	ExecutionStatusInProgress string = "InProgress"
-
-	ExecutionTriggerManual   string = "Manual"
-	ExecutionTriggerEvent    string = "Event"
-	ExecutionTriggerSchedule string = "Schedule"
-
-	// The task has been persisted in db but not submitted to Jobservice
-	TaskStatusInitialized string = "Initialized"
-	TaskStatusPending     string = "Pending"
-	TaskStatusInProgress  string = "InProgress"
-	TaskStatusSucceed     string = "Succeed"
-	TaskStatusFailed      string = "Failed"
-	TaskStatusStopped     string = "Stopped"
-)
-
-// ExecutionPropsName defines the names of fields of Execution
-var ExecutionPropsName = ExecutionFieldsName{
-	ID:         "ID",
-	PolicyID:   "PolicyID",
-	Status:     "Status",
-	StatusText: "StatusText",
-	Total:      "Total",
-	Failed:     "Failed",
-	Succeed:    "Succeed",
-	InProgress: "InProgress",
-	Stopped:    "Stopped",
-	Trigger:    "Trigger",
-	StartTime:  "StartTime",
-	EndTime:    "EndTime",
-}
-
-// ExecutionFieldsName defines the props of Execution
-type ExecutionFieldsName struct {
-	ID         string
-	PolicyID   string
-	Status     string
-	StatusText string
-	Total      string
-	Failed     string
-	Succeed    string
-	InProgress string
-	Stopped    string
-	Trigger    string
-	StartTime  string
-	EndTime    string
-}
-
-// Execution holds information about once replication execution.
-type Execution struct {
-	ID         int64             `orm:"pk;auto;column(id)" json:"id"`
-	PolicyID   int64             `orm:"column(policy_id)" json:"policy_id"`
-	Status     string            `orm:"column(status)" json:"status"`
-	StatusText string            `orm:"column(status_text)" json:"status_text"`
-	Total      int               `orm:"column(total)" json:"total"`
-	Failed     int               `orm:"column(failed)" json:"failed"`
-	Succeed    int               `orm:"column(succeed)" json:"succeed"`
-	InProgress int               `orm:"column(in_progress)" json:"in_progress"`
-	Stopped    int               `orm:"column(stopped)" json:"stopped"`
-	Trigger    model.TriggerType `orm:"column(trigger)" json:"trigger"`
-	StartTime  time.Time         `orm:"column(start_time)" json:"start_time"`
-	EndTime    time.Time         `orm:"column(end_time)" json:"end_time"`
-}
-
-// TaskPropsName defines the names of fields of Task
-var TaskPropsName = TaskFieldsName{
-	ID:           "ID",
-	ExecutionID:  "ExecutionID",
-	ResourceType: "ResourceType",
-	SrcResource:  "SrcResource",
-	DstResource:  "DstResource",
-	JobID:        "JobID",
-	Status:       "Status",
-	StartTime:    "StartTime",
-	EndTime:      "EndTime",
-}
-
-// TaskFieldsName defines the props of Task
-type TaskFieldsName struct {
-	ID           string
-	ExecutionID  string
-	ResourceType string
-	SrcResource  string
-	DstResource  string
-	JobID        string
-	Status       string
-	StartTime    string
-	EndTime      string
-}
-
-// Task represent the tasks in one execution.
-type Task struct {
-	ID             int64     `orm:"pk;auto;column(id)" json:"id"`
-	ExecutionID    int64     `orm:"column(execution_id)" json:"execution_id"`
-	ResourceType   string    `orm:"column(resource_type)" json:"resource_type"`
-	SrcResource    string    `orm:"column(src_resource)" json:"src_resource"`
-	DstResource    string    `orm:"column(dst_resource)" json:"dst_resource"`
-	Operation      string    `orm:"column(operation)" json:"operation"`
-	JobID          string    `orm:"column(job_id)" json:"job_id"`
-	Status         string    `orm:"column(status)" json:"status"`
-	StatusRevision int64     `orm:"column(status_revision)"`
-	StartTime      time.Time `orm:"column(start_time)" json:"start_time"`
-	EndTime        time.Time `orm:"column(end_time)" json:"end_time,omitempty"`
-}
-
-// TableName is required by by beego orm to map Execution to table replication_execution
-func (r *Execution) TableName() string {
-	return ExecutionTable
-}
-
-// TableName is required by by beego orm to map Task to table replication_task
-func (r *Task) TableName() string {
-	return TaskTable
-}
-
-// ExecutionQuery holds the query conditions for replication executions
-type ExecutionQuery struct {
-	PolicyID int64
-	Statuses []string
-	Trigger  string
-	Pagination
-}
-
-// TaskQuery holds the query conditions for replication task
-type TaskQuery struct {
-	ExecutionID  int64
-	JobID        string
-	Statuses     []string
-	ResourceType string
-	Pagination
-}
-
-// TaskStat holds statistics of task by status
-type TaskStat struct {
-	Status string `orm:"column(status)"`
-	C      int    `orm:"column(c)"`
-}

src/replication/dao/models/schedule_job.go

@@ -1,40 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// TODO rename the package name to model
-
-package models
-
-import "time"
-
-// ScheduleJob is the persistent model for the schedule job which is
-// used as a scheduler
-type ScheduleJob struct {
-	ID           int64     `orm:"pk;auto;column(id)" json:"id"`
-	PolicyID     int64     `orm:"column(policy_id)" json:"policy_id"`
-	JobID        string    `orm:"column(job_id)" json:"job_id"`
-	Status       string    `orm:"column(status)" json:"status"`
-	CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"`
-	UpdateTime   time.Time `orm:"column(update_time);auto_now" json:"update_time"`
-}
-
-// TableName is required by by beego orm to map the object to the database table
-func (s *ScheduleJob) TableName() string {
-	return "replication_schedule_job"
-}
-
-// ScheduleJobQuery is the query used to list schedule jobs
-type ScheduleJobQuery struct {
-	PolicyID int64
-}

src/replication/dao/schedule_job.go

@@ -1,92 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package dao
-
-import (
-	"time"
-
-	"github.com/astaxie/beego/orm"
-	"github.com/goharbor/harbor/src/common/dao"
-	"github.com/goharbor/harbor/src/replication/dao/models"
-)
-
-// ScheduleJob is the DAO for schedule job
-var ScheduleJob ScheduleJobDAO = &scheduleJobDAO{}
-
-// ScheduleJobDAO ...
-type ScheduleJobDAO interface {
-	Add(*models.ScheduleJob) (int64, error)
-	Get(int64) (*models.ScheduleJob, error)
-	Update(*models.ScheduleJob, ...string) error
-	Delete(int64) error
-	List(...*models.ScheduleJobQuery) ([]*models.ScheduleJob, error)
-}
-
-type scheduleJobDAO struct{}
-
-func (s *scheduleJobDAO) Add(sj *models.ScheduleJob) (int64, error) {
-	now := time.Now()
-	sj.CreationTime = now
-	sj.UpdateTime = now
-	return dao.GetOrmer().Insert(sj)
-}
-
-func (s *scheduleJobDAO) Get(id int64) (*models.ScheduleJob, error) {
-	sj := &models.ScheduleJob{
-		ID: id,
-	}
-	if err := dao.GetOrmer().Read(sj); err != nil {
-		if err == orm.ErrNoRows {
-			return nil, nil
-		}
-		return nil, err
-	}
-	return sj, nil
-}
-
-func (s *scheduleJobDAO) Update(sj *models.ScheduleJob, props ...string) error {
-	if sj.UpdateTime.IsZero() {
-		now := time.Now()
-		sj.UpdateTime = now
-		if len(props) > 0 {
-			props = append(props, "UpdateTime")
-		}
-	}
-
-	_, err := dao.GetOrmer().Update(sj, props...)
-	return err
-}
-
-func (s *scheduleJobDAO) Delete(id int64) error {
-	_, err := dao.GetOrmer().Delete(&models.ScheduleJob{
-		ID: id,
-	})
-	return err
-}
-
-func (s *scheduleJobDAO) List(query ...*models.ScheduleJobQuery) ([]*models.ScheduleJob, error) {
-	qs := dao.GetOrmer().QueryTable(&models.ScheduleJob{})
-	if len(query) > 0 && query[0] != nil {
-		if query[0].PolicyID > 0 {
-			qs = qs.Filter("PolicyID", query[0].PolicyID)
-		}
-	}
-	sjs := []*models.ScheduleJob{}
-	_, err := qs.All(&sjs)
-	if err != nil {
-		return nil, err
-	}
-	return sjs, nil
-}

src/replication/dao/schedule_job_test.go

@@ -1,81 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package dao
-
-import (
-	"testing"
-
-	"github.com/goharbor/harbor/src/replication/dao/models"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-var sjID int64
-
-func TestAddScheduleJob(t *testing.T) {
-	sj := &models.ScheduleJob{
-		PolicyID: 1,
-		JobID:    "uuid",
-		Status:   "running",
-	}
-	id, err := ScheduleJob.Add(sj)
-	require.Nil(t, err)
-	sjID = id
-}
-
-func TestUpdateScheduleJob(t *testing.T) {
-	err := ScheduleJob.Update(&models.ScheduleJob{
-		ID:     sjID,
-		Status: "success",
-	}, "Status")
-	require.Nil(t, err)
-}
-
-func TestGetScheduleJob(t *testing.T) {
-	sj, err := ScheduleJob.Get(sjID)
-	require.Nil(t, err)
-	assert.Equal(t, int64(1), sj.PolicyID)
-	assert.Equal(t, "success", sj.Status)
-}
-
-func TestListScheduleJobs(t *testing.T) {
-	// nil query
-	sjs, err := ScheduleJob.List()
-	require.Nil(t, err)
-	assert.Equal(t, 1, len(sjs))
-
-	// query
-	sjs, err = ScheduleJob.List(&models.ScheduleJobQuery{
-		PolicyID: 1,
-	})
-	require.Nil(t, err)
-	assert.Equal(t, 1, len(sjs))
-
-	// query
-	sjs, err = ScheduleJob.List(&models.ScheduleJobQuery{
-		PolicyID: 2,
-	})
-	require.Nil(t, err)
-	assert.Equal(t, 0, len(sjs))
-}
-
-func TestDeleteScheduleJob(t *testing.T) {
-	err := ScheduleJob.Delete(sjID)
-	require.Nil(t, err)
-
-	sj, err := ScheduleJob.Get(sjID)
-	require.Nil(t, err)
-	assert.Nil(t, sj)
-}

src/replication/event/handler.go

@@ -17,12 +17,14 @@ package event
 import (
 	"errors"
 	"fmt"
+	"github.com/goharbor/harbor/src/lib/orm"
+	"github.com/goharbor/harbor/src/pkg/task"
 
 	commonthttp "github.com/goharbor/harbor/src/common/http"
+	"github.com/goharbor/harbor/src/controller/replication"
 	"github.com/goharbor/harbor/src/lib/log"
 	"github.com/goharbor/harbor/src/replication/config"
 	"github.com/goharbor/harbor/src/replication/model"
-	"github.com/goharbor/harbor/src/replication/operation"
 	"github.com/goharbor/harbor/src/replication/policy"
 	"github.com/goharbor/harbor/src/replication/registry"
 	"github.com/goharbor/harbor/src/replication/util"
@@ -34,18 +36,18 @@ type Handler interface {
 }
 
 // NewHandler ...
-func NewHandler(policyCtl policy.Controller, registryMgr registry.Manager, opCtl operation.Controller) Handler {
+func NewHandler(policyCtl policy.Controller, registryMgr registry.Manager) Handler {
 	return &handler{
 		policyCtl:   policyCtl,
 		registryMgr: registryMgr,
-		opCtl:       opCtl,
+		ctl:         replication.Ctl,
 	}
 }
 
 type handler struct {
 	policyCtl   policy.Controller
 	registryMgr registry.Manager
-	opCtl       operation.Controller
+	ctl         replication.Controller
 }
 
 func (h *handler) Handle(event *Event) error {
@@ -76,7 +78,7 @@ func (h *handler) Handle(event *Event) error {
 		if err := PopulateRegistries(h.registryMgr, policy); err != nil {
 			return err
 		}
-		id, err := h.opCtl.StartReplication(policy, event.Resource, model.TriggerTypeEventBased)
+		id, err := h.ctl.Start(orm.Context(), policy, event.Resource, task.ExecutionTriggerEvent)
 		if err != nil {
 			return err
 		}

src/replication/event/handler_test.go

@@ -15,43 +15,18 @@
 package event
 
 import (
-	"github.com/goharbor/harbor/src/lib/q"
+	"github.com/goharbor/harbor/src/common/dao"
+	"github.com/goharbor/harbor/src/testing/controller/replication"
+	"github.com/goharbor/harbor/src/testing/mock"
 	"testing"
 
+	"github.com/goharbor/harbor/src/lib/q"
 	"github.com/goharbor/harbor/src/replication/config"
-	"github.com/goharbor/harbor/src/replication/dao/models"
 	"github.com/goharbor/harbor/src/replication/model"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
-type fakedOperationController struct{}
-
-func (f *fakedOperationController) StartReplication(policy *model.Policy, resource *model.Resource, trigger model.TriggerType) (int64, error) {
-	return 1, nil
-}
-func (f *fakedOperationController) StopReplication(int64) error {
-	return nil
-}
-func (f *fakedOperationController) ListExecutions(...*models.ExecutionQuery) (int64, []*models.Execution, error) {
-	return 0, nil, nil
-}
-func (f *fakedOperationController) GetExecution(id int64) (*models.Execution, error) {
-	return nil, nil
-}
-func (f *fakedOperationController) ListTasks(...*models.TaskQuery) (int64, []*models.Task, error) {
-	return 0, nil, nil
-}
-func (f *fakedOperationController) GetTask(id int64) (*models.Task, error) {
-	return nil, nil
-}
-func (f *fakedOperationController) UpdateTaskStatus(id int64, status string, statusRevision int64, statusCondition ...string) error {
-	return nil
-}
-func (f *fakedOperationController) GetTaskLog(int64) ([]byte, error) {
-	return nil, nil
-}
-
 type fakedPolicyController struct{}
 
 func (f *fakedPolicyController) Create(*model.Policy) (int64, error) {
@@ -236,10 +211,16 @@ func TestGetRelatedPolicies(t *testing.T) {
 }
 
 func TestHandle(t *testing.T) {
+	dao.PrepareTestForPostgresSQL()
 	config.Config = &config.Configuration{}
-	handler := NewHandler(&fakedPolicyController{},
+	ctl := &replication.Controller{}
-		&fakedRegistryManager{},
+	ctl.On("Start", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil)
-		&fakedOperationController{})
+	handler := &handler{
+		policyCtl:   &fakedPolicyController{},
+		registryMgr: &fakedRegistryManager{},
+		ctl:         ctl,
+	}
+
 	// nil event
 	err := handler.Handle(nil)
 	require.NotNil(t, err)

src/replication/operation/controller.go

@@ -1,243 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package operation
-
-import (
-	"fmt"
-	"time"
-
-	"github.com/goharbor/harbor/src/common/job"
-	hjob "github.com/goharbor/harbor/src/jobservice/job"
-	"github.com/goharbor/harbor/src/lib/log"
-	"github.com/goharbor/harbor/src/replication/dao/models"
-	"github.com/goharbor/harbor/src/replication/model"
-	"github.com/goharbor/harbor/src/replication/operation/execution"
-	"github.com/goharbor/harbor/src/replication/operation/flow"
-	"github.com/goharbor/harbor/src/replication/operation/scheduler"
-)
-
-// Controller handles the replication-related operations: start,
-// stop, query, etc.
-type Controller interface {
-	// trigger is used to specify what this replication is triggered by
-	StartReplication(policy *model.Policy, resource *model.Resource, trigger model.TriggerType) (int64, error)
-	StopReplication(int64) error
-	ListExecutions(...*models.ExecutionQuery) (int64, []*models.Execution, error)
-	GetExecution(int64) (*models.Execution, error)
-	ListTasks(...*models.TaskQuery) (int64, []*models.Task, error)
-	GetTask(int64) (*models.Task, error)
-	UpdateTaskStatus(id int64, status string, statusRevision int64, statusCondition ...string) error
-	GetTaskLog(int64) ([]byte, error)
-}
-
-const (
-	maxReplicators = 1024
-)
-
-// NewController returns a controller implementation
-func NewController(js job.Client) Controller {
-	ctl := &controller{
-		replicators:  make(chan struct{}, maxReplicators),
-		executionMgr: execution.NewDefaultManager(),
-		scheduler:    scheduler.NewScheduler(js),
-		flowCtl:      flow.NewController(),
-	}
-	for i := 0; i < maxReplicators; i++ {
-		ctl.replicators <- struct{}{}
-	}
-	return ctl
-}
-
-type controller struct {
-	replicators  chan struct{}
-	flowCtl      flow.Controller
-	executionMgr execution.Manager
-	scheduler    scheduler.Scheduler
-}
-
-func (c *controller) StartReplication(policy *model.Policy, resource *model.Resource, trigger model.TriggerType) (int64, error) {
-	if !policy.Enabled {
-		return 0, fmt.Errorf("the policy %d is disabled", policy.ID)
-	}
-	if len(trigger) == 0 {
-		trigger = model.TriggerTypeManual
-	}
-	id, err := createExecution(c.executionMgr, policy.ID, trigger)
-	if err != nil {
-		return 0, err
-	}
-	// control the count of concurrent replication requests
-	log.Debugf("waiting for the available replicator ...")
-	<-c.replicators
-	log.Debugf("got an available replicator, starting the replication ...")
-	go func() {
-		defer func() {
-			c.replicators <- struct{}{}
-		}()
-		flow := c.createFlow(id, policy, resource)
-		if n, err := c.flowCtl.Start(flow); err != nil {
-			// only update the execution when got error.
-			// if got no error, it will be updated automatically
-			// when listing the execution records
-			if e := c.executionMgr.Update(&models.Execution{
-				ID:         id,
-				Status:     models.ExecutionStatusFailed,
-				StatusText: err.Error(),
-				Total:      n,
-				Failed:     n,
-			}, "Status", "StatusText", "Total", "Failed"); e != nil {
-				log.Errorf("failed to update the execution %d: %v", id, e)
-			}
-			log.Errorf("the execution %d failed: %v", id, err)
-		}
-	}()
-	return id, nil
-}
-
-// create different replication flows according to the input parameters
-func (c *controller) createFlow(executionID int64, policy *model.Policy, resource *model.Resource) flow.Flow {
-	// replicate the deletion operation, so create a deletion flow
-	if resource != nil && resource.Deleted {
-		return flow.NewDeletionFlow(c.executionMgr, c.scheduler, executionID, policy, resource)
-	}
-	resources := []*model.Resource{}
-	if resource != nil {
-		resources = append(resources, resource)
-	}
-	return flow.NewCopyFlow(c.executionMgr, c.scheduler, executionID, policy, resources...)
-}
-
-func (c *controller) StopReplication(executionID int64) error {
-	_, tasks, err := c.ListTasks(&models.TaskQuery{
-		ExecutionID: executionID,
-	})
-	if err != nil {
-		return err
-	}
-	// no tasks, just set its status to "stopped"
-	if len(tasks) == 0 {
-		execution, err := c.executionMgr.Get(executionID)
-		if err != nil {
-			return err
-		}
-		if execution == nil {
-			return fmt.Errorf("the execution %d not found", executionID)
-		}
-		if execution.Status != models.ExecutionStatusInProgress {
-			log.Debugf("the execution %d isn't in progress, no need to stop", executionID)
-			return nil
-		}
-		if err = c.executionMgr.Update(&models.Execution{
-			ID:      executionID,
-			Status:  models.ExecutionStatusStopped,
-			EndTime: time.Now(),
-		}, models.ExecutionPropsName.Status, models.ExecutionPropsName.EndTime); err != nil {
-			return err
-		}
-		log.Debugf("the status of execution %d is set to stopped", executionID)
-	}
-	// got tasks, stopping the tasks one by one
-	for _, task := range tasks {
-		if isTaskInFinalStatus(task) {
-			log.Debugf("the task %d(job ID: %s) is in final status, its status is %s, skip", task.ID, task.JobID, task.Status)
-			continue
-		}
-		if err = c.scheduler.Stop(task.JobID); err != nil {
-			isStatusBehindError, ok := err.(*job.StatusBehindError)
-			if ok {
-				status := isStatusBehindError.Status()
-				switch hjob.Status(status) {
-				case hjob.ErrorStatus:
-					status = models.TaskStatusFailed
-				case hjob.SuccessStatus:
-					status = models.TaskStatusSucceed
-				}
-				e := c.executionMgr.UpdateTask(&models.Task{
-					ID:     task.ID,
-					Status: status,
-				}, "Status")
-				if e != nil {
-					log.Errorf("failed to update the status the task %d(job ID: %s): %v", task.ID, task.JobID, e)
-				} else {
-					log.Debugf("got status behind error for task %d, update it's status to %s directly", task.ID, status)
-				}
-				continue
-			}
-			if err == job.ErrJobNotFound {
-				e := c.executionMgr.UpdateTask(&models.Task{
-					ID:     task.ID,
-					Status: models.ExecutionStatusStopped,
-				}, "Status")
-				if e != nil {
-					log.Errorf("failed to update the status the task %d(job ID: %s): %v", task.ID, task.JobID, e)
-				} else {
-					log.Debugf("got job not found error for task %d, update it's status to %s directly", task.ID, models.ExecutionStatusStopped)
-				}
-				continue
-			}
-			log.Errorf("failed to stop the task %d(job ID: %s): %v", task.ID, task.JobID, err)
-			continue
-		}
-		log.Debugf("the stop request for task %d(job ID: %s) sent", task.ID, task.JobID)
-	}
-	return nil
-}
-
-func isTaskInFinalStatus(task *models.Task) bool {
-	if task == nil {
-		return false
-	}
-	switch task.Status {
-	case models.TaskStatusSucceed,
-		models.TaskStatusStopped,
-		models.TaskStatusFailed:
-		return true
-	}
-	return false
-}
-
-func (c *controller) ListExecutions(query ...*models.ExecutionQuery) (int64, []*models.Execution, error) {
-	return c.executionMgr.List(query...)
-}
-func (c *controller) GetExecution(executionID int64) (*models.Execution, error) {
-	return c.executionMgr.Get(executionID)
-}
-func (c *controller) ListTasks(query ...*models.TaskQuery) (int64, []*models.Task, error) {
-	return c.executionMgr.ListTasks(query...)
-}
-func (c *controller) GetTask(id int64) (*models.Task, error) {
-	return c.executionMgr.GetTask(id)
-}
-func (c *controller) UpdateTaskStatus(id int64, status string, statusRevision int64, statusCondition ...string) error {
-	return c.executionMgr.UpdateTaskStatus(id, status, statusRevision, statusCondition...)
-}
-func (c *controller) GetTaskLog(taskID int64) ([]byte, error) {
-	return c.executionMgr.GetTaskLog(taskID)
-}
-
-// create the execution record in database
-func createExecution(mgr execution.Manager, policyID int64, trigger model.TriggerType) (int64, error) {
-	id, err := mgr.Create(&models.Execution{
-		PolicyID:  policyID,
-		Trigger:   trigger,
-		Status:    models.ExecutionStatusInProgress,
-		StartTime: time.Now(),
-	})
-	if err != nil {
-		return 0, fmt.Errorf("failed to create the execution record for replication based on policy %d: %v", policyID, err)
-	}
-	log.Debugf("an execution record for replication based on the policy %d created: %d", policyID, id)
-	return id, nil
-}

src/replication/operation/controller_test.go

@@ -1,390 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package operation
-
-import (
-	"io"
-	"os"
-	"testing"
-
-	"github.com/docker/distribution"
-	"github.com/goharbor/harbor/src/replication/adapter"
-	"github.com/goharbor/harbor/src/replication/config"
-	"github.com/goharbor/harbor/src/replication/dao/models"
-	"github.com/goharbor/harbor/src/replication/model"
-	"github.com/goharbor/harbor/src/replication/operation/flow"
-	"github.com/goharbor/harbor/src/replication/operation/scheduler"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-type fakedExecutionManager struct{}
-
-func (f *fakedExecutionManager) Create(*models.Execution) (int64, error) {
-	return 1, nil
-}
-func (f *fakedExecutionManager) List(...*models.ExecutionQuery) (int64, []*models.Execution, error) {
-	return 1, []*models.Execution{
-		{
-			ID: 1,
-		},
-	}, nil
-}
-func (f *fakedExecutionManager) Get(int64) (*models.Execution, error) {
-	return &models.Execution{
-		ID: 1,
-	}, nil
-}
-func (f *fakedExecutionManager) Update(*models.Execution, ...string) error {
-	return nil
-}
-func (f *fakedExecutionManager) Remove(int64) error {
-	return nil
-}
-func (f *fakedExecutionManager) RemoveAll(int64) error {
-	return nil
-}
-func (f *fakedExecutionManager) CreateTask(*models.Task) (int64, error) {
-	return 1, nil
-}
-func (f *fakedExecutionManager) ListTasks(...*models.TaskQuery) (int64, []*models.Task, error) {
-	return 1, []*models.Task{
-		{
-			ID: 1,
-		},
-	}, nil
-}
-func (f *fakedExecutionManager) GetTask(int64) (*models.Task, error) {
-	return &models.Task{
-		ID: 1,
-	}, nil
-}
-func (f *fakedExecutionManager) UpdateTask(*models.Task, ...string) error {
-	return nil
-}
-func (f *fakedExecutionManager) UpdateTaskStatus(int64, string, int64, ...string) error {
-	return nil
-}
-func (f *fakedExecutionManager) RemoveTask(int64) error {
-	return nil
-}
-func (f *fakedExecutionManager) RemoveAllTasks(int64) error {
-	return nil
-}
-func (f *fakedExecutionManager) GetTaskLog(int64) ([]byte, error) {
-	return []byte("message"), nil
-}
-
-type fakedScheduler struct{}
-
-func (f *fakedScheduler) Preprocess(src []*model.Resource, dst []*model.Resource) ([]*scheduler.ScheduleItem, error) {
-	items := make([]*scheduler.ScheduleItem, 0)
-	for i, res := range src {
-		items = append(items, &scheduler.ScheduleItem{
-			SrcResource: res,
-			DstResource: dst[i],
-		})
-	}
-	return items, nil
-}
-func (f *fakedScheduler) Schedule(items []*scheduler.ScheduleItem) ([]*scheduler.ScheduleResult, error) {
-	results := make([]*scheduler.ScheduleResult, 0)
-	for _, item := range items {
-		results = append(results, &scheduler.ScheduleResult{
-			TaskID: item.TaskID,
-			Error:  nil,
-		})
-	}
-	return results, nil
-}
-func (f *fakedScheduler) Stop(id string) error {
-	return nil
-}
-
-type fakedFactory struct {
-}
-
-func (fakedFactory) Create(*model.Registry) (adapter.Adapter, error) {
-	return &fakedAdapter{}, nil
-}
-
-func (fakedFactory) AdapterPattern() *model.AdapterPattern {
-	return nil
-}
-
-type fakedAdapter struct{}
-
-func (f *fakedAdapter) Info() (*model.RegistryInfo, error) {
-	return &model.RegistryInfo{
-		Type: model.RegistryTypeHarbor,
-		SupportedResourceTypes: []model.ResourceType{
-			model.ResourceTypeImage,
-			model.ResourceTypeChart,
-		},
-		SupportedTriggers: []model.TriggerType{model.TriggerTypeManual},
-	}, nil
-}
-
-func (f *fakedAdapter) PrepareForPush([]*model.Resource) error {
-	return nil
-}
-func (f *fakedAdapter) HealthCheck() (model.HealthStatus, error) {
-	return model.Healthy, nil
-}
-func (f *fakedAdapter) FetchImages(namespace []string, filters []*model.Filter) ([]*model.Resource, error) {
-	return []*model.Resource{
-		{
-			Type: model.ResourceTypeImage,
-			Metadata: &model.ResourceMetadata{
-				Repository: &model.Repository{
-					Name: "library/hello-world",
-				},
-				Vtags: []string{"latest"},
-			},
-			Override: false,
-		},
-	}, nil
-}
-
-func (f *fakedAdapter) ManifestExist(repository, reference string) (exist bool, digest string, err error) {
-	return false, "", nil
-}
-func (f *fakedAdapter) PullManifest(repository, reference string, accepttedMediaTypes []string) (manifest distribution.Manifest, digest string, err error) {
-	return nil, "", nil
-}
-func (f *fakedAdapter) PushManifest(repository, reference, mediaType string, payload []byte) error {
-	return nil
-}
-func (f *fakedAdapter) DeleteManifest(repository, digest string) error {
-	return nil
-}
-func (f *fakedAdapter) BlobExist(repository, digest string) (exist bool, err error) {
-	return false, nil
-}
-func (f *fakedAdapter) PullBlob(repository, digest string) (size int64, blob io.ReadCloser, err error) {
-	return 0, nil, nil
-}
-func (f *fakedAdapter) PushBlob(repository, digest string, size int64, blob io.Reader) error {
-	return nil
-}
-func (f *fakedAdapter) FetchCharts(namespaces []string, filters []*model.Filter) ([]*model.Resource, error) {
-	return []*model.Resource{
-		{
-			Type: model.ResourceTypeChart,
-			Metadata: &model.ResourceMetadata{
-				Repository: &model.Repository{
-					Name: "library/harbor",
-				},
-				Vtags: []string{"0.2.0"},
-			},
-		},
-	}, nil
-}
-func (f *fakedAdapter) ChartExist(name, version string) (bool, error) {
-	return false, nil
-}
-func (f *fakedAdapter) DownloadChart(name, version string) (io.ReadCloser, error) {
-	return nil, nil
-}
-func (f *fakedAdapter) UploadChart(name, version string, chart io.Reader) error {
-	return nil
-}
-func (f *fakedAdapter) DeleteChart(name, version string) error {
-	return nil
-}
-
-var ctl *controller
-
-func TestMain(m *testing.M) {
-	ctl = &controller{
-		replicators:  make(chan struct{}, 1),
-		executionMgr: &fakedExecutionManager{},
-		scheduler:    &fakedScheduler{},
-		flowCtl:      flow.NewController(),
-	}
-	ctl.replicators <- struct{}{}
-	os.Exit(m.Run())
-}
-
-func TestStartReplication(t *testing.T) {
-	err := adapter.RegisterFactory(model.RegistryTypeHarbor, new(fakedFactory))
-	require.Nil(t, err)
-	config.Config = &config.Configuration{}
-
-	// policy is disabled
-	policy := &model.Policy{
-		SrcRegistry: &model.Registry{
-			Type: model.RegistryTypeHarbor,
-		},
-		DestRegistry: &model.Registry{
-			Type: model.RegistryTypeHarbor,
-		},
-	}
-	resource := &model.Resource{
-		Type: model.ResourceTypeImage,
-		Metadata: &model.ResourceMetadata{
-			Repository: &model.Repository{
-				Name: "library/hello-world",
-			},
-			Vtags: []string{"1.0", "2.0"},
-		},
-	}
-	_, err = ctl.StartReplication(policy, resource, model.TriggerTypeEventBased)
-	require.NotNil(t, err)
-
-	// replicate resource deletion
-	policy = &model.Policy{
-		SrcRegistry: &model.Registry{
-			Type: model.RegistryTypeHarbor,
-		},
-		DestRegistry: &model.Registry{
-			Type: model.RegistryTypeHarbor,
-		},
-		Enabled: true,
-	}
-	resource = &model.Resource{
-		Type: model.ResourceTypeImage,
-		Metadata: &model.ResourceMetadata{
-			Repository: &model.Repository{
-				Name: "library/hello-world",
-			},
-			Vtags: []string{"1.0", "2.0"},
-		},
-		Deleted: true,
-	}
-	id, err := ctl.StartReplication(policy, resource, model.TriggerTypeEventBased)
-	require.Nil(t, err)
-	assert.Equal(t, int64(1), id)
-
-	// replicate resource copy
-	policy = &model.Policy{
-		SrcRegistry: &model.Registry{
-			Type: model.RegistryTypeHarbor,
-		},
-		DestRegistry: &model.Registry{
-			Type: model.RegistryTypeHarbor,
-		},
-		Enabled: true,
-	}
-	resource = &model.Resource{
-		Type: model.ResourceTypeImage,
-		Metadata: &model.ResourceMetadata{
-			Repository: &model.Repository{
-				Name: "library/hello-world",
-			},
-			Vtags: []string{"1.0", "2.0"},
-		},
-		Deleted: false,
-	}
-	id, err = ctl.StartReplication(policy, resource, model.TriggerTypeEventBased)
-	require.Nil(t, err)
-	assert.Equal(t, int64(1), id)
-
-	// nil resource
-	policy = &model.Policy{
-		SrcRegistry: &model.Registry{
-			Type: model.RegistryTypeHarbor,
-		},
-		DestRegistry: &model.Registry{
-			Type: model.RegistryTypeHarbor,
-		},
-		Enabled: true,
-	}
-	id, err = ctl.StartReplication(policy, nil, model.TriggerTypeEventBased)
-	require.Nil(t, err)
-	assert.Equal(t, int64(1), id)
-}
-
-func TestStopReplication(t *testing.T) {
-	err := ctl.StopReplication(1)
-	require.Nil(t, err)
-}
-
-func TestListExecutions(t *testing.T) {
-	n, executions, err := ctl.ListExecutions()
-	require.Nil(t, err)
-	assert.Equal(t, int64(1), n)
-	assert.Equal(t, int64(1), executions[0].ID)
-}
-
-func TestGetExecution(t *testing.T) {
-	execution, err := ctl.GetExecution(1)
-	require.Nil(t, err)
-	assert.Equal(t, int64(1), execution.ID)
-}
-
-func TestListTasks(t *testing.T) {
-	n, tasks, err := ctl.ListTasks()
-	require.Nil(t, err)
-	assert.Equal(t, int64(1), n)
-	assert.Equal(t, int64(1), tasks[0].ID)
-}
-
-func TestGetTask(t *testing.T) {
-	task, err := ctl.GetTask(1)
-	require.Nil(t, err)
-	assert.Equal(t, int64(1), task.ID)
-}
-
-func TestUpdateTaskStatus(t *testing.T) {
-	err := ctl.UpdateTaskStatus(1, "running", 1)
-	require.Nil(t, err)
-}
-
-func TestGetTaskLog(t *testing.T) {
-	log, err := ctl.GetTaskLog(1)
-	require.Nil(t, err)
-	assert.Equal(t, "message", string(log))
-}
-
-func TestIsTaskRunning(t *testing.T) {
-	cases := []struct {
-		task          *models.Task
-		isFinalStatus bool
-	}{
-		{
-			task:          nil,
-			isFinalStatus: false,
-		},
-		{
-			task: &models.Task{
-				Status: models.TaskStatusSucceed,
-			},
-			isFinalStatus: true,
-		},
-		{
-			task: &models.Task{
-				Status: models.TaskStatusFailed,
-			},
-			isFinalStatus: true,
-		},
-		{
-			task: &models.Task{
-				Status: models.TaskStatusStopped,
-			},
-			isFinalStatus: true,
-		},
-		{
-			task: &models.Task{
-				Status: models.TaskStatusInProgress,
-			},
-			isFinalStatus: false,
-		},
-	}
-
-	for _, c := range cases {
-		assert.Equal(t, c.isFinalStatus, isTaskInFinalStatus(c.task))
-	}
-}

src/replication/operation/execution/execution.go

@@ -1,182 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package execution
-
-import (
-	"fmt"
-
-	"github.com/goharbor/harbor/src/core/utils"
-	"github.com/goharbor/harbor/src/replication/dao"
-	"github.com/goharbor/harbor/src/replication/dao/models"
-)
-
-// Manager manages the executions
-type Manager interface {
-	// Create a new execution
-	Create(*models.Execution) (int64, error)
-	// List the summaries of executions
-	List(...*models.ExecutionQuery) (int64, []*models.Execution, error)
-	// Get the specified execution
-	Get(int64) (*models.Execution, error)
-	// Update the data of the specified execution, the "props" are the
-	// properties of execution that need to be updated
-	Update(execution *models.Execution, props ...string) error
-	// Remove the execution specified by the ID
-	Remove(int64) error
-	// Remove all executions of one policy specified by the policy ID
-	RemoveAll(int64) error
-	// Create a task
-	CreateTask(*models.Task) (int64, error)
-	// List the tasks according to the query
-	ListTasks(...*models.TaskQuery) (int64, []*models.Task, error)
-	// Get one specified task
-	GetTask(int64) (*models.Task, error)
-	// Update the task, the "props" are the properties of task
-	// that need to be updated, it cannot include "status". If
-	// you want to update the status, use "UpdateTaskStatus" instead
-	UpdateTask(task *models.Task, props ...string) error
-	// UpdateTaskStatus only updates the task status. If "statusCondition"
-	// presents, only the tasks whose status equal to "statusCondition"
-	// will be updated
-	UpdateTaskStatus(taskID int64, status string, statusRevision int64, statusCondition ...string) error
-	// Remove one task specified by task ID
-	RemoveTask(int64) error
-	// Remove all tasks of one execution specified by the execution ID
-	RemoveAllTasks(int64) error
-	// Get the log of one specific task
-	GetTaskLog(int64) ([]byte, error)
-}
-
-// DefaultManager ..
-type DefaultManager struct {
-}
-
-// NewDefaultManager ...
-func NewDefaultManager() Manager {
-	return &DefaultManager{}
-}
-
-// Create a new execution
-func (dm *DefaultManager) Create(execution *models.Execution) (int64, error) {
-	return dao.AddExecution(execution)
-}
-
-// List the summaries of executions
-func (dm *DefaultManager) List(queries ...*models.ExecutionQuery) (int64, []*models.Execution, error) {
-	total, err := dao.GetTotalOfExecutions(queries...)
-	if err != nil {
-		return 0, nil, err
-	}
-
-	executions, err := dao.GetExecutions(queries...)
-	if err != nil {
-		return 0, nil, err
-	}
-	return total, executions, nil
-}
-
-// Get the specified execution
-func (dm *DefaultManager) Get(id int64) (*models.Execution, error) {
-	return dao.GetExecution(id)
-}
-
-// Update ...
-func (dm *DefaultManager) Update(execution *models.Execution, props ...string) error {
-	n, err := dao.UpdateExecution(execution, props...)
-	if err != nil {
-		return err
-	}
-	if n == 0 {
-		return fmt.Errorf("Execution not found error: %d ", execution.ID)
-	}
-	return nil
-}
-
-// Remove the execution specified by the ID
-func (dm *DefaultManager) Remove(id int64) error {
-	return dao.DeleteExecution(id)
-}
-
-// RemoveAll executions of one policy specified by the policy ID
-func (dm *DefaultManager) RemoveAll(policyID int64) error {
-	return dao.DeleteAllExecutions(policyID)
-}
-
-// CreateTask used to create a task
-func (dm *DefaultManager) CreateTask(task *models.Task) (int64, error) {
-	return dao.AddTask(task)
-}
-
-// ListTasks list the tasks according to the query
-func (dm *DefaultManager) ListTasks(queries ...*models.TaskQuery) (int64, []*models.Task, error) {
-	total, err := dao.GetTotalOfTasks(queries...)
-	if err != nil {
-		return 0, nil, err
-	}
-
-	tasks, err := dao.GetTasks(queries...)
-	if err != nil {
-		return 0, nil, err
-	}
-	return total, tasks, nil
-}
-
-// GetTask get one specified task
-func (dm *DefaultManager) GetTask(id int64) (*models.Task, error) {
-	return dao.GetTask(id)
-}
-
-// UpdateTask ...
-func (dm *DefaultManager) UpdateTask(task *models.Task, props ...string) error {
-	n, err := dao.UpdateTask(task, props...)
-	if err != nil {
-		return err
-	}
-	if n == 0 {
-		return fmt.Errorf("Task not found error: %d ", task.ID)
-	}
-	return nil
-}
-
-// UpdateTaskStatus ...
-func (dm *DefaultManager) UpdateTaskStatus(taskID int64, status string, statusRevision int64, statusCondition ...string) error {
-	if _, err := dao.UpdateTaskStatus(taskID, status, statusRevision, statusCondition...); err != nil {
-		return err
-	}
-	return nil
-}
-
-// RemoveTask remove one task specified by task ID
-func (dm *DefaultManager) RemoveTask(id int64) error {
-	return dao.DeleteTask(id)
-}
-
-// RemoveAllTasks of one execution specified by the execution ID
-func (dm *DefaultManager) RemoveAllTasks(executionID int64) error {
-	return dao.DeleteAllTasks(executionID)
-}
-
-// GetTaskLog get the log of one specific task
-func (dm *DefaultManager) GetTaskLog(taskID int64) ([]byte, error) {
-	task, err := dao.GetTask(taskID)
-	if err != nil {
-		return nil, err
-	}
-	if task == nil {
-		return nil, fmt.Errorf("Task not found %d ", taskID)
-	}
-
-	return utils.GetJobServiceClient().GetJobLog(task.JobID)
-}

src/replication/operation/execution/execution_test.go

@@ -1,135 +0,0 @@
-package execution
-
-import (
-	"os"
-	"testing"
-	"time"
-
-	"github.com/goharbor/harbor/src/common/dao"
-	"github.com/goharbor/harbor/src/replication/dao/models"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-var executionManager = NewDefaultManager()
-
-func TestMain(m *testing.M) {
-	dao.PrepareTestForPostgresSQL()
-	os.Exit(m.Run())
-}
-
-func TestMethodOfExecutionManager(t *testing.T) {
-	execution := &models.Execution{
-		PolicyID:   11209,
-		Status:     "InProgress",
-		StatusText: "None",
-		Total:      12,
-		Failed:     0,
-		Succeed:    7,
-		InProgress: 5,
-		Stopped:    0,
-		Trigger:    "Event",
-		StartTime:  time.Now(),
-	}
-
-	defer func() {
-		executionManager.RemoveAll(execution.PolicyID)
-	}()
-
-	// Create
-	id, err := executionManager.Create(execution)
-	require.Nil(t, err)
-
-	// List
-	query := &models.ExecutionQuery{
-		Statuses: []string{"InProgress", "Failed"},
-		Pagination: models.Pagination{
-			Page: 1,
-			Size: 10,
-		},
-	}
-	count, executions, err := executionManager.List(query)
-	require.Nil(t, err)
-	assert.Equal(t, int64(1), count)
-	assert.Equal(t, 1, len(executions))
-
-	// Get
-	_, err = executionManager.Get(id)
-	require.Nil(t, err)
-
-	// Update
-	executionNew := &models.Execution{
-		ID:         id,
-		Status:     "Failed",
-		Succeed:    12,
-		InProgress: 0,
-		EndTime:    time.Now(),
-	}
-	err = executionManager.Update(executionNew, models.ExecutionPropsName.Status, models.ExecutionPropsName.Succeed, models.ExecutionPropsName.InProgress,
-		models.ExecutionPropsName.EndTime)
-	require.Nil(t, err)
-
-	// Remove
-	require.Nil(t, executionManager.Remove(id))
-}
-
-func TestMethodOfTaskManager(t *testing.T) {
-	now := time.Now()
-	task := &models.Task{
-		ExecutionID:    112200,
-		ResourceType:   "resourceType1",
-		SrcResource:    "srcResource1",
-		DstResource:    "dstResource1",
-		JobID:          "jobID1",
-		Status:         "Initialized",
-		StatusRevision: 1,
-		StartTime:      now,
-	}
-
-	defer func() {
-		executionManager.RemoveAllTasks(task.ExecutionID)
-	}()
-
-	// CreateTask
-	id, err := executionManager.CreateTask(task)
-	require.Nil(t, err)
-
-	// ListTasks
-	query := &models.TaskQuery{
-		ResourceType: "resourceType1",
-		Pagination: models.Pagination{
-			Page: 1,
-			Size: 10,
-		},
-	}
-	count, tasks, err := executionManager.ListTasks(query)
-	require.Nil(t, err)
-	assert.Equal(t, 1, len(tasks))
-	assert.Equal(t, int64(1), count)
-
-	// GetTask
-	_, err = executionManager.GetTask(id)
-	require.Nil(t, err)
-
-	// UpdateTask
-	taskNew := &models.Task{
-		ID:          id,
-		SrcResource: "srcResourceChanged",
-	}
-	err = executionManager.UpdateTask(taskNew, models.TaskPropsName.SrcResource)
-	require.Nil(t, err)
-	taskUpdate, _ := executionManager.GetTask(id)
-	assert.Equal(t, taskNew.SrcResource, taskUpdate.SrcResource)
-
-	// UpdateTaskStatus
-	err = executionManager.UpdateTaskStatus(id, models.TaskStatusSucceed, 1, models.TaskStatusInitialized)
-	require.Nil(t, err)
-	taskUpdate, _ = executionManager.GetTask(id)
-	assert.Equal(t, models.TaskStatusSucceed, taskUpdate.Status)
-
-	// Remove
-	require.Nil(t, executionManager.RemoveTask(id))
-
-	// RemoveAll
-	require.Nil(t, executionManager.RemoveAll(id))
-}

src/replication/operation/flow/copy.go

@@ -1,109 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package flow
-
-import (
-	"time"
-
-	"github.com/goharbor/harbor/src/lib/log"
-	"github.com/goharbor/harbor/src/replication/dao/models"
-	"github.com/goharbor/harbor/src/replication/model"
-	"github.com/goharbor/harbor/src/replication/operation/execution"
-	"github.com/goharbor/harbor/src/replication/operation/scheduler"
-)
-
-type copyFlow struct {
-	executionID  int64
-	resources    []*model.Resource
-	policy       *model.Policy
-	executionMgr execution.Manager
-	scheduler    scheduler.Scheduler
-}
-
-// NewCopyFlow returns an instance of the copy flow which replicates the resources from
-// the source registry to the destination registry. If the parameter "resources" isn't provided,
-// will fetch the resources first
-func NewCopyFlow(executionMgr execution.Manager, scheduler scheduler.Scheduler,
-	executionID int64, policy *model.Policy, resources ...*model.Resource) Flow {
-	return &copyFlow{
-		executionMgr: executionMgr,
-		scheduler:    scheduler,
-		executionID:  executionID,
-		policy:       policy,
-		resources:    resources,
-	}
-}
-
-func (c *copyFlow) Run(interface{}) (int, error) {
-	srcAdapter, dstAdapter, err := initialize(c.policy)
-	if err != nil {
-		return 0, err
-	}
-	var srcResources []*model.Resource
-	if len(c.resources) > 0 {
-		srcResources, err = filterResources(c.resources, c.policy.Filters)
-	} else {
-		srcResources, err = fetchResources(srcAdapter, c.policy)
-	}
-	if err != nil {
-		return 0, err
-	}
-
-	isStopped, err := isExecutionStopped(c.executionMgr, c.executionID)
-	if err != nil {
-		return 0, err
-	}
-	if isStopped {
-		log.Debugf("the execution %d is stopped, stop the flow", c.executionID)
-		return 0, nil
-	}
-
-	if len(srcResources) == 0 {
-		markExecutionSuccess(c.executionMgr, c.executionID, "no resources need to be replicated")
-		log.Infof("no resources need to be replicated for the execution %d, skip", c.executionID)
-		return 0, nil
-	}
-
-	srcResources = assembleSourceResources(srcResources, c.policy)
-	dstResources := assembleDestinationResources(srcResources, c.policy)
-
-	if err = prepareForPush(dstAdapter, dstResources); err != nil {
-		return 0, err
-	}
-	items, err := preprocess(c.scheduler, srcResources, dstResources)
-	if err != nil {
-		return 0, err
-	}
-	if err = createTasks(c.executionMgr, c.executionID, items); err != nil {
-		return 0, err
-	}
-
-	return schedule(c.scheduler, c.executionMgr, items)
-}
-
-// mark the execution as success in database
-func markExecutionSuccess(mgr execution.Manager, id int64, message string) {
-	err := mgr.Update(
-		&models.Execution{
-			ID:         id,
-			Status:     models.ExecutionStatusSucceed,
-			StatusText: message,
-			EndTime:    time.Now(),
-		}, "Status", "StatusText", "EndTime")
-	if err != nil {
-		log.Errorf("failed to update the execution %d: %v", id, err)
-		return
-	}
-}

src/replication/operation/flow/copy_test.go

@@ -1,36 +0,0 @@
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package flow
-
-import (
-	"testing"
-
-	"github.com/goharbor/harbor/src/replication/model"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestRunOfCopyFlow(t *testing.T) {
-	scheduler := &fakedScheduler{}
-	executionMgr := &fakedExecutionManager{}
-	policy := &model.Policy{
-		SrcRegistry: &model.Registry{
-			Type: model.RegistryTypeHarbor,
-		},
-		DestRegistry: &model.Registry{
-			Type: model.RegistryTypeHarbor,
-		},
-	}
-	flow := NewCopyFlow(executionMgr, scheduler, 1, policy)
-	n, err := flow.Run(nil)
-	require.Nil(t, err)
-	assert.Equal(t, 2, n)
-}

src/replication/operation/flow/deletion.go

@@ -1,68 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package flow
-
-import (
-	"github.com/goharbor/harbor/src/lib/log"
-	"github.com/goharbor/harbor/src/replication/model"
-	"github.com/goharbor/harbor/src/replication/operation/execution"
-	"github.com/goharbor/harbor/src/replication/operation/scheduler"
-)
-
-type deletionFlow struct {
-	executionID  int64
-	policy       *model.Policy
-	executionMgr execution.Manager
-	scheduler    scheduler.Scheduler
-	resources    []*model.Resource
-}
-
-// NewDeletionFlow returns an instance of the delete flow which deletes the resources
-// on the destination registry
-func NewDeletionFlow(executionMgr execution.Manager, scheduler scheduler.Scheduler,
-	executionID int64, policy *model.Policy, resources ...*model.Resource) Flow {
-	return &deletionFlow{
-		executionMgr: executionMgr,
-		scheduler:    scheduler,
-		executionID:  executionID,
-		policy:       policy,
-		resources:    resources,
-	}
-}
-
-func (d *deletionFlow) Run(interface{}) (int, error) {
-	srcResources, err := filterResources(d.resources, d.policy.Filters)
-	if err != nil {
-		return 0, err
-	}
-	if len(srcResources) == 0 {
-		markExecutionSuccess(d.executionMgr, d.executionID, "no resources need to be replicated")
-		log.Infof("no resources need to be replicated for the execution %d, skip", d.executionID)
-		return 0, nil
-	}
-
-	srcResources = assembleSourceResources(srcResources, d.policy)
-	dstResources := assembleDestinationResources(srcResources, d.policy)
-
-	items, err := preprocess(d.scheduler, srcResources, dstResources)
-	if err != nil {
-		return 0, err
-	}
-	if err = createTasks(d.executionMgr, d.executionID, items); err != nil {
-		return 0, err
-	}
-
-	return schedule(d.scheduler, d.executionMgr, items)
-}

src/replication/operation/flow/stage_test.go

@@ -1,427 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package flow
-
-import (
-	"io"
-	"os"
-	"testing"
-
-	"github.com/docker/distribution"
-	"github.com/goharbor/harbor/src/replication/adapter"
-	"github.com/goharbor/harbor/src/replication/config"
-	"github.com/goharbor/harbor/src/replication/dao/models"
-	"github.com/goharbor/harbor/src/replication/model"
-	"github.com/goharbor/harbor/src/replication/operation/scheduler"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-type fakedFactory struct {
-}
-
-func (fakedFactory) Create(*model.Registry) (adapter.Adapter, error) {
-	return &fakedAdapter{}, nil
-}
-
-func (fakedFactory) AdapterPattern() *model.AdapterPattern {
-	return nil
-}
-
-type fakedAdapter struct{}
-
-func (f *fakedAdapter) Info() (*model.RegistryInfo, error) {
-	return &model.RegistryInfo{
-		Type: model.RegistryTypeHarbor,
-		SupportedResourceTypes: []model.ResourceType{
-			model.ResourceTypeImage,
-			model.ResourceTypeChart,
-		},
-		SupportedTriggers: []model.TriggerType{model.TriggerTypeManual},
-	}, nil
-}
-
-func (f *fakedAdapter) PrepareForPush([]*model.Resource) error {
-	return nil
-}
-func (f *fakedAdapter) HealthCheck() (model.HealthStatus, error) {
-	return model.Healthy, nil
-}
-func (f *fakedAdapter) FetchArtifacts(filters []*model.Filter) ([]*model.Resource, error) {
-	return []*model.Resource{
-		{
-			Type: model.ResourceTypeImage,
-			Metadata: &model.ResourceMetadata{
-				Repository: &model.Repository{
-					Name: "library/hello-world",
-				},
-				Vtags: []string{"latest"},
-			},
-			Override: false,
-		},
-	}, nil
-}
-
-func (f *fakedAdapter) ManifestExist(repository, reference string) (exist bool, digest string, err error) {
-	return false, "", nil
-}
-func (f *fakedAdapter) PullManifest(repository, reference string, accepttedMediaTypes ...string) (manifest distribution.Manifest, digest string, err error) {
-	return nil, "", nil
-}
-func (f *fakedAdapter) PushManifest(repository, reference, mediaType string, payload []byte) (string, error) {
-	return "", nil
-}
-func (f *fakedAdapter) DeleteManifest(repository, digest string) error {
-	return nil
-}
-func (f *fakedAdapter) BlobExist(repository, digest string) (exist bool, err error) {
-	return false, nil
-}
-func (f *fakedAdapter) PullBlob(repository, digest string) (size int64, blob io.ReadCloser, err error) {
-	return 0, nil, nil
-}
-func (f *fakedAdapter) PushBlob(repository, digest string, size int64, blob io.Reader) error {
-	return nil
-}
-func (f *fakedAdapter) DeleteTag(repository, tag string) error {
-	return nil
-}
-func (f *fakedAdapter) FetchCharts(filters []*model.Filter) ([]*model.Resource, error) {
-	return []*model.Resource{
-		{
-			Type: model.ResourceTypeChart,
-			Metadata: &model.ResourceMetadata{
-				Repository: &model.Repository{
-					Name: "library/harbor",
-				},
-				Vtags: []string{"0.2.0"},
-			},
-		},
-	}, nil
-}
-func (f *fakedAdapter) ChartExist(name, version string) (bool, error) {
-	return false, nil
-}
-func (f *fakedAdapter) DownloadChart(name, version string) (io.ReadCloser, error) {
-	return nil, nil
-}
-func (f *fakedAdapter) UploadChart(name, version string, chart io.Reader) error {
-	return nil
-}
-func (f *fakedAdapter) DeleteChart(name, version string) error {
-	return nil
-}
-
-type fakedScheduler struct{}
-
-func (f *fakedScheduler) Preprocess(src []*model.Resource, dst []*model.Resource) ([]*scheduler.ScheduleItem, error) {
-	items := []*scheduler.ScheduleItem{}
-	for i, res := range src {
-		items = append(items, &scheduler.ScheduleItem{
-			SrcResource: res,
-			DstResource: dst[i],
-		})
-	}
-	return items, nil
-}
-func (f *fakedScheduler) Schedule(items []*scheduler.ScheduleItem) ([]*scheduler.ScheduleResult, error) {
-	results := []*scheduler.ScheduleResult{}
-	for _, item := range items {
-		results = append(results, &scheduler.ScheduleResult{
-			TaskID: item.TaskID,
-			Error:  nil,
-		})
-	}
-	return results, nil
-}
-func (f *fakedScheduler) Stop(id string) error {
-	return nil
-}
-
-type fakedExecutionManager struct {
-	taskID int64
-}
-
-func (f *fakedExecutionManager) Create(*models.Execution) (int64, error) {
-	return 1, nil
-}
-func (f *fakedExecutionManager) List(...*models.ExecutionQuery) (int64, []*models.Execution, error) {
-	return 0, nil, nil
-}
-func (f *fakedExecutionManager) Get(int64) (*models.Execution, error) {
-	return &models.Execution{}, nil
-}
-func (f *fakedExecutionManager) Update(*models.Execution, ...string) error {
-	return nil
-}
-func (f *fakedExecutionManager) Remove(int64) error {
-	return nil
-}
-func (f *fakedExecutionManager) RemoveAll(int64) error {
-	return nil
-}
-func (f *fakedExecutionManager) CreateTask(*models.Task) (int64, error) {
-	f.taskID++
-	id := f.taskID
-	return id, nil
-}
-func (f *fakedExecutionManager) ListTasks(...*models.TaskQuery) (int64, []*models.Task, error) {
-	return 0, nil, nil
-}
-func (f *fakedExecutionManager) GetTask(int64) (*models.Task, error) {
-	return nil, nil
-}
-func (f *fakedExecutionManager) UpdateTask(*models.Task, ...string) error {
-	return nil
-}
-func (f *fakedExecutionManager) UpdateTaskStatus(int64, string, int64, ...string) error {
-	return nil
-}
-func (f *fakedExecutionManager) RemoveTask(int64) error {
-	return nil
-}
-func (f *fakedExecutionManager) RemoveAllTasks(int64) error {
-	return nil
-}
-func (f *fakedExecutionManager) GetTaskLog(int64) ([]byte, error) {
-	return nil, nil
-}
-
-func TestMain(m *testing.M) {
-	url := "https://registry.harbor.local"
-	config.Config = &config.Configuration{
-		CoreURL: url,
-	}
-	if err := adapter.RegisterFactory(model.RegistryTypeHarbor, new(fakedFactory)); err != nil {
-		os.Exit(1)
-	}
-	os.Exit(m.Run())
-}
-
-func TestFetchResources(t *testing.T) {
-	adapter := &fakedAdapter{}
-	policy := &model.Policy{}
-	resources, err := fetchResources(adapter, policy)
-	require.Nil(t, err)
-	assert.Equal(t, 2, len(resources))
-}
-
-func TestFilterResources(t *testing.T) {
-	resources := []*model.Resource{
-		{
-			Type: model.ResourceTypeImage,
-			Metadata: &model.ResourceMetadata{
-				Repository: &model.Repository{
-					Name: "library/hello-world",
-				},
-				Artifacts: []*model.Artifact{
-					{
-						Tags: []string{"latest"},
-					},
-				},
-			},
-			Deleted:  true,
-			Override: true,
-		},
-		{
-			Type: model.ResourceTypeChart,
-			Metadata: &model.ResourceMetadata{
-				Repository: &model.Repository{
-					Name: "library/harbor",
-				},
-				Artifacts: []*model.Artifact{
-					{
-						Tags: []string{"0.2.0"},
-					},
-					{
-						Tags: []string{"0.3.0"},
-					},
-				},
-			},
-			Deleted:  true,
-			Override: true,
-		},
-		{
-			Type: model.ResourceTypeChart,
-			Metadata: &model.ResourceMetadata{
-				Repository: &model.Repository{
-					Name: "library/mysql",
-				},
-				Artifacts: []*model.Artifact{
-					{
-						Tags: []string{"1.0"},
-					},
-				},
-			},
-			Deleted:  true,
-			Override: true,
-		},
-	}
-	filters := []*model.Filter{
-		{
-			Type:  model.FilterTypeResource,
-			Value: model.ResourceTypeChart,
-		},
-		{
-			Type:  model.FilterTypeName,
-			Value: "library/*",
-		},
-		{
-			Type:  model.FilterTypeName,
-			Value: "library/harbor",
-		},
-		{
-			Type:  model.FilterTypeTag,
-			Value: "0.2.?",
-		},
-	}
-	res, err := filterResources(resources, filters)
-	require.Nil(t, err)
-	assert.Equal(t, 1, len(res))
-	assert.Equal(t, "library/harbor", res[0].Metadata.Repository.Name)
-	assert.Equal(t, 1, len(res[0].Metadata.Artifacts))
-	assert.Equal(t, "0.2.0", res[0].Metadata.Artifacts[0].Tags[0])
-}
-
-func TestAssembleSourceResources(t *testing.T) {
-	resources := []*model.Resource{
-		{
-			Type: model.ResourceTypeChart,
-			Metadata: &model.ResourceMetadata{
-				Repository: &model.Repository{
-					Name: "library/hello-world",
-				},
-				Vtags: []string{"latest"},
-			},
-			Override: false,
-		},
-	}
-	policy := &model.Policy{
-		SrcRegistry: &model.Registry{
-			ID: 1,
-		},
-	}
-	res := assembleSourceResources(resources, policy)
-	assert.Equal(t, 1, len(res))
-	assert.Equal(t, int64(1), res[0].Registry.ID)
-}
-
-func TestAssembleDestinationResources(t *testing.T) {
-	resources := []*model.Resource{
-		{
-			Type: model.ResourceTypeChart,
-			Metadata: &model.ResourceMetadata{
-				Repository: &model.Repository{
-					Name: "library/hello-world",
-				},
-				Vtags: []string{"latest"},
-			},
-			Override: false,
-		},
-	}
-	policy := &model.Policy{
-		DestRegistry:  &model.Registry{},
-		DestNamespace: "test",
-		Override:      true,
-	}
-	res := assembleDestinationResources(resources, policy)
-	assert.Equal(t, 1, len(res))
-	assert.Equal(t, model.ResourceTypeChart, res[0].Type)
-	assert.Equal(t, "test/hello-world", res[0].Metadata.Repository.Name)
-	assert.Equal(t, 1, len(res[0].Metadata.Vtags))
-	assert.Equal(t, "latest", res[0].Metadata.Vtags[0])
-}
-
-func TestPreprocess(t *testing.T) {
-	scheduler := &fakedScheduler{}
-	srcResources := []*model.Resource{
-		{
-			Type: model.ResourceTypeChart,
-			Metadata: &model.ResourceMetadata{
-				Repository: &model.Repository{
-					Name: "library/hello-world",
-				},
-				Vtags: []string{"latest"},
-			},
-			Override: false,
-		},
-	}
-	dstResources := []*model.Resource{
-		{
-			Type: model.ResourceTypeChart,
-			Metadata: &model.ResourceMetadata{
-				Repository: &model.Repository{
-					Name: "test/hello-world",
-				},
-				Vtags: []string{"latest"},
-			},
-			Override: false,
-		},
-	}
-	items, err := preprocess(scheduler, srcResources, dstResources)
-	require.Nil(t, err)
-	assert.Equal(t, 1, len(items))
-}
-
-func TestCreateTasks(t *testing.T) {
-	mgr := &fakedExecutionManager{}
-	items := []*scheduler.ScheduleItem{
-		{
-			SrcResource: &model.Resource{},
-			DstResource: &model.Resource{},
-		},
-	}
-	err := createTasks(mgr, 1, items)
-	require.Nil(t, err)
-	assert.Equal(t, int64(1), items[0].TaskID)
-}
-
-func TestSchedule(t *testing.T) {
-	sched := &fakedScheduler{}
-	mgr := &fakedExecutionManager{}
-	items := []*scheduler.ScheduleItem{
-		{
-			SrcResource: &model.Resource{},
-			DstResource: &model.Resource{},
-			TaskID:      1,
-		},
-	}
-	n, err := schedule(sched, mgr, items)
-	require.Nil(t, err)
-	assert.Equal(t, 1, n)
-}
-
-func TestReplaceNamespace(t *testing.T) {
-	// empty namespace
-	repository := "c"
-	namespace := ""
-	result := replaceNamespace(repository, namespace)
-	assert.Equal(t, "c", result)
-	// repository contains no "/"
-	repository = "c"
-	namespace = "n"
-	result = replaceNamespace(repository, namespace)
-	assert.Equal(t, "n/c", result)
-	// repository contains only one "/"
-	repository = "b/c"
-	namespace = "n"
-	result = replaceNamespace(repository, namespace)
-	assert.Equal(t, "n/c", result)
-	// repository contains more than one "/"
-	repository = "a/b/c"
-	namespace = "n"
-	result = replaceNamespace(repository, namespace)
-	assert.Equal(t, "n/c", result)
-}

src/replication/operation/hook/task.go

@@ -1,47 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package hook
-
-import (
-	"github.com/goharbor/harbor/src/jobservice/job"
-	"github.com/goharbor/harbor/src/replication/dao/models"
-	"github.com/goharbor/harbor/src/replication/operation"
-)
-
-// UpdateTask update the status of the task
-func UpdateTask(ctl operation.Controller, id int64, status string, statusRevision int64) error {
-	jobStatus := job.Status(status)
-	// convert the job status to task status
-	s := ""
-	preStatus := []string{}
-	switch jobStatus {
-	case job.PendingStatus:
-		s = models.TaskStatusPending
-		preStatus = append(preStatus, models.TaskStatusInitialized)
-	case job.ScheduledStatus, job.RunningStatus:
-		s = models.TaskStatusInProgress
-		preStatus = append(preStatus, models.TaskStatusInitialized, models.TaskStatusPending)
-	case job.StoppedStatus:
-		s = models.TaskStatusStopped
-		preStatus = append(preStatus, models.TaskStatusInitialized, models.TaskStatusPending, models.TaskStatusInProgress)
-	case job.ErrorStatus:
-		s = models.TaskStatusFailed
-		preStatus = append(preStatus, models.TaskStatusInitialized, models.TaskStatusPending, models.TaskStatusInProgress)
-	case job.SuccessStatus:
-		s = models.TaskStatusSucceed
-		preStatus = append(preStatus, models.TaskStatusInitialized, models.TaskStatusPending, models.TaskStatusInProgress)
-	}
-	return ctl.UpdateTaskStatus(id, s, statusRevision, preStatus...)
-}

src/replication/operation/hook/task_test.go

@@ -1,94 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package hook
-
-import (
-	"testing"
-
-	"github.com/goharbor/harbor/src/jobservice/job"
-	"github.com/goharbor/harbor/src/replication/dao/models"
-	"github.com/goharbor/harbor/src/replication/model"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-type fakedOperationController struct {
-	status string
-}
-
-func (f *fakedOperationController) StartReplication(*model.Policy, *model.Resource, model.TriggerType) (int64, error) {
-	return 0, nil
-}
-func (f *fakedOperationController) StopReplication(int64) error {
-	return nil
-}
-func (f *fakedOperationController) ListExecutions(...*models.ExecutionQuery) (int64, []*models.Execution, error) {
-	return 0, nil, nil
-}
-func (f *fakedOperationController) GetExecution(int64) (*models.Execution, error) {
-	return nil, nil
-}
-func (f *fakedOperationController) ListTasks(...*models.TaskQuery) (int64, []*models.Task, error) {
-	return 0, nil, nil
-}
-func (f *fakedOperationController) GetTask(int64) (*models.Task, error) {
-	return nil, nil
-}
-func (f *fakedOperationController) UpdateTaskStatus(id int64, status string, statusRevision int64, statusCondition ...string) error {
-	f.status = status
-	return nil
-}
-func (f *fakedOperationController) GetTaskLog(int64) ([]byte, error) {
-	return nil, nil
-}
-
-func TestUpdateTask(t *testing.T) {
-	mgr := &fakedOperationController{}
-	cases := []struct {
-		inputStatus    string
-		expectedStatus string
-	}{
-		{
-			inputStatus:    job.PendingStatus.String(),
-			expectedStatus: models.TaskStatusPending,
-		},
-		{
-			inputStatus:    job.ScheduledStatus.String(),
-			expectedStatus: models.TaskStatusInProgress,
-		},
-		{
-			inputStatus:    job.RunningStatus.String(),
-			expectedStatus: models.TaskStatusInProgress,
-		},
-		{
-			inputStatus:    job.StoppedStatus.String(),
-			expectedStatus: models.TaskStatusStopped,
-		},
-		{
-			inputStatus:    job.ErrorStatus.String(),
-			expectedStatus: models.TaskStatusFailed,
-		},
-		{
-			inputStatus:    job.SuccessStatus.String(),
-			expectedStatus: models.TaskStatusSucceed,
-		},
-	}
-
-	for _, c := range cases {
-		err := UpdateTask(mgr, 1, c.inputStatus, 1)
-		require.Nil(t, err)
-		assert.Equal(t, c.expectedStatus, mgr.status)
-	}
-}

src/replication/operation/scheduler/scheduler.go

@@ -1,141 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package scheduler
-
-import (
-	"encoding/json"
-	"errors"
-	"fmt"
-
-	cjob "github.com/goharbor/harbor/src/common/job"
-	"github.com/goharbor/harbor/src/common/job/models"
-	"github.com/goharbor/harbor/src/jobservice/job"
-	"github.com/goharbor/harbor/src/replication/config"
-	"github.com/goharbor/harbor/src/replication/model"
-)
-
-type defaultScheduler struct {
-	client cjob.Client
-}
-
-// NewScheduler returns an instance of Scheduler
-func NewScheduler(js cjob.Client) Scheduler {
-	return &defaultScheduler{
-		client: js,
-	}
-}
-
-// ScheduleItem is an item that can be scheduled
-type ScheduleItem struct {
-	TaskID      int64 // used as the param in the hook
-	SrcResource *model.Resource
-	DstResource *model.Resource
-}
-
-// ScheduleResult is the result of the schedule for one item
-type ScheduleResult struct {
-	TaskID int64
-	JobID  string
-	Error  error
-}
-
-// Scheduler schedules
-type Scheduler interface {
-	// Preprocess the resources and returns the item list that can be scheduled
-	Preprocess([]*model.Resource, []*model.Resource) ([]*ScheduleItem, error)
-	// Schedule the items. If got error when scheduling one of the items,
-	// the error should be put in the corresponding ScheduleResult and the
-	// returning error of this function should be nil
-	Schedule([]*ScheduleItem) ([]*ScheduleResult, error)
-	// Stop the job specified by ID
-	Stop(id string) error
-}
-
-// Preprocess the resources and returns the item list that can be scheduled
-func (d *defaultScheduler) Preprocess(srcResources []*model.Resource, destResources []*model.Resource) ([]*ScheduleItem, error) {
-	if len(srcResources) != len(destResources) {
-		err := errors.New("srcResources has different length with destResources")
-		return nil, err
-	}
-	var items []*ScheduleItem
-	for index, srcResource := range srcResources {
-		destResource := destResources[index]
-		item := &ScheduleItem{
-			SrcResource: srcResource,
-			DstResource: destResource,
-		}
-		items = append(items, item)
-
-	}
-	return items, nil
-}
-
-// Schedule transfer the tasks to jobs,and then submit these jobs to job service.
-func (d *defaultScheduler) Schedule(items []*ScheduleItem) ([]*ScheduleResult, error) {
-	var results []*ScheduleResult
-	for _, item := range items {
-		result := &ScheduleResult{
-			TaskID: item.TaskID,
-		}
-		if item.TaskID == 0 {
-			result.Error = errors.New("some tasks do not have a ID")
-			results = append(results, result)
-			continue
-		}
-		j := &models.JobData{
-			Metadata: &models.JobMetadata{
-				JobKind: job.KindGeneric,
-			},
-			StatusHook: fmt.Sprintf("%s/service/notifications/jobs/replication/task/%d", config.Config.CoreURL, item.TaskID),
-		}
-
-		j.Name = job.Replication
-		src, err := json.Marshal(item.SrcResource)
-		if err != nil {
-			result.Error = err
-			results = append(results, result)
-			continue
-		}
-		dest, err := json.Marshal(item.DstResource)
-		if err != nil {
-			result.Error = err
-			results = append(results, result)
-			continue
-		}
-		j.Parameters = map[string]interface{}{
-			"src_resource": string(src),
-			"dst_resource": string(dest),
-		}
-		id, joberr := d.client.SubmitJob(j)
-		if joberr != nil {
-			result.Error = joberr
-			results = append(results, result)
-			continue
-		}
-		result.JobID = id
-		results = append(results, result)
-	}
-	return results, nil
-}
-
-// Stop the transfer job
-func (d *defaultScheduler) Stop(id string) error {
-	err := d.client.PostAction(id, string(job.StopCommand))
-	if err != nil {
-		return err
-	}
-	return nil
-
-}

src/replication/operation/scheduler/scheduler_test.go

@@ -1,79 +0,0 @@
-package scheduler
-
-import (
-	"encoding/json"
-	"testing"
-
-	"github.com/goharbor/harbor/src/common/job/models"
-	"github.com/goharbor/harbor/src/jobservice/job"
-	"github.com/goharbor/harbor/src/replication/model"
-)
-
-var scheduler = &defaultScheduler{
-	client: TestClient{},
-}
-
-type TestClient struct {
-}
-
-func (client TestClient) SubmitJob(*models.JobData) (string, error) {
-	return "submited-uuid", nil
-}
-func (client TestClient) GetJobLog(uuid string) ([]byte, error) {
-	return []byte("job log"), nil
-}
-func (client TestClient) PostAction(uuid, action string) error {
-	return nil
-}
-func (client TestClient) GetExecutions(uuid string) ([]job.Stats, error) {
-	return nil, nil
-}
-
-func TestPreprocess(t *testing.T) {
-	items, err := generateData()
-	if err != nil {
-		t.Error(err)
-	}
-	for _, item := range items {
-		content, err := json.Marshal(item)
-		if err != nil {
-			t.Error(err)
-		}
-		t.Log(string(content))
-	}
-
-}
-
-func TestStop(t *testing.T) {
-	err := scheduler.Stop("id")
-	if err != nil {
-		t.Error(err)
-	}
-}
-
-func generateData() ([]*ScheduleItem, error) {
-	srcResource := &model.Resource{
-		Metadata: &model.ResourceMetadata{
-			Repository: &model.Repository{
-				Name: "namespace1",
-			},
-			Vtags: []string{"latest"},
-		},
-		Registry: &model.Registry{
-			Credential: &model.Credential{},
-		},
-	}
-	destResource := &model.Resource{
-		Metadata: &model.ResourceMetadata{
-			Repository: &model.Repository{
-				Name: "namespace2",
-			},
-			Vtags: []string{"v1", "v2"},
-		},
-		Registry: &model.Registry{
-			Credential: &model.Credential{},
-		},
-	}
-	items, err := scheduler.Preprocess([]*model.Resource{srcResource}, []*model.Resource{destResource})
-	return items, err
-}

src/replication/policy/controller/controller.go

@@ -17,20 +17,25 @@ package controller
 import (
 	"fmt"
 
-	"github.com/goharbor/harbor/src/common/job"
+	"github.com/goharbor/harbor/src/jobservice/job"
 	"github.com/goharbor/harbor/src/lib/log"
+	"github.com/goharbor/harbor/src/lib/orm"
+	"github.com/goharbor/harbor/src/pkg/scheduler"
 	"github.com/goharbor/harbor/src/replication/model"
 	"github.com/goharbor/harbor/src/replication/policy"
 	"github.com/goharbor/harbor/src/replication/policy/manager"
-	"github.com/goharbor/harbor/src/replication/policy/scheduler"
+)
+
+// const definitions
+const (
+	CallbackFuncName = "REPLICATION_CALLBACK"
 )
 
 // NewController returns a policy controller which can CURD and schedule policies
-func NewController(js job.Client) policy.Controller {
+func NewController() policy.Controller {
 	mgr := manager.NewDefaultManager()
-	scheduler := scheduler.NewScheduler(js)
 	ctl := &controller{
-		scheduler: scheduler,
+		scheduler: scheduler.Sched,
 	}
 	ctl.Controller = mgr
 	return ctl
@@ -47,10 +52,7 @@ func (c *controller) Create(policy *model.Policy) (int64, error) {
 		return 0, err
 	}
 	if isScheduledTrigger(policy) {
-		// TODO: need a way to show the schedule status to users
+		if _, err = c.scheduler.Schedule(orm.Context(), job.Replication, id, "", policy.Trigger.Settings.Cron, CallbackFuncName, id); err != nil {
-		// maybe we can add a property "schedule status" for
-		// listing policy API
-		if err = c.scheduler.Schedule(id, policy.Trigger.Settings.Cron); err != nil {
 			log.Errorf("failed to schedule the policy %d: %v", id, err)
 		}
 	}
@@ -72,7 +74,7 @@ func (c *controller) Update(policy *model.Policy) error {
 	// need to reschedule the policy
 	// unschedule first if needed
 	if isScheduledTrigger(origin) {
-		if err = c.scheduler.Unschedule(origin.ID); err != nil {
+		if err = c.scheduler.UnScheduleByVendor(orm.Context(), job.Replication, origin.ID); err != nil {
 			return fmt.Errorf("failed to unschedule the policy %d: %v", origin.ID, err)
 		}
 	}
@@ -82,7 +84,7 @@ func (c *controller) Update(policy *model.Policy) error {
 	}
 	// schedule again if needed
 	if isScheduledTrigger(policy) {
-		if err = c.scheduler.Schedule(policy.ID, policy.Trigger.Settings.Cron); err != nil {
+		if _, err = c.scheduler.Schedule(orm.Context(), job.Replication, policy.ID, "", policy.Trigger.Settings.Cron, CallbackFuncName, policy.ID); err != nil {
 			return fmt.Errorf("failed to schedule the policy %d: %v", policy.ID, err)
 		}
 	}
@@ -98,7 +100,7 @@ func (c *controller) Remove(policyID int64) error {
 		return fmt.Errorf("policy %d not found", policyID)
 	}
 	if isScheduledTrigger(policy) {
-		if err = c.scheduler.Unschedule(policyID); err != nil {
+		if err = c.scheduler.UnScheduleByVendor(orm.Context(), job.Replication, policyID); err != nil {
 			return err
 		}
 	}

src/replication/policy/controller/controller_test.go

@@ -17,7 +17,10 @@ package controller
 import (
 	"testing"
 
+	"github.com/goharbor/harbor/src/common/dao"
 	"github.com/goharbor/harbor/src/replication/model"
+	"github.com/goharbor/harbor/src/testing/mock"
+	"github.com/goharbor/harbor/src/testing/pkg/scheduler"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -45,20 +48,6 @@ func (f *fakedPolicyController) Remove(int64) error {
 	return nil
 }
 
-type fakedScheduler struct {
-	scheduled   bool
-	unscheduled bool
-}
-
-func (f *fakedScheduler) Schedule(policyID int64, cron string) error {
-	f.scheduled = true
-	return nil
-}
-func (f *fakedScheduler) Unschedule(policyID int64) error {
-	f.unscheduled = true
-	return nil
-}
-
 func TestIsScheduledTrigger(t *testing.T) {
 	cases := []struct {
 		policy   *model.Policy
@@ -224,7 +213,8 @@ func TestIsScheduleTriggerChanged(t *testing.T) {
 }
 
 func TestCreate(t *testing.T) {
-	scheduler := &fakedScheduler{}
+	dao.PrepareTestForPostgresSQL()
+	scheduler := &scheduler.Scheduler{}
 	ctl := &controller{
 		scheduler: scheduler,
 	}
@@ -233,9 +223,10 @@ func TestCreate(t *testing.T) {
 	// not scheduled trigger
 	_, err := ctl.Create(&model.Policy{})
 	require.Nil(t, err)
-	assert.False(t, scheduler.scheduled)
 
 	// scheduled trigger
+	scheduler.On("Schedule", mock.Anything, mock.Anything,
+		mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil)
 	_, err = ctl.Create(&model.Policy{
 		Enabled: true,
 		Trigger: &model.Trigger{
@@ -246,11 +237,11 @@ func TestCreate(t *testing.T) {
 		},
 	})
 	require.Nil(t, err)
-	assert.True(t, scheduler.scheduled)
+	scheduler.AssertExpectations(t)
 }
 
 func TestUpdate(t *testing.T) {
-	scheduler := &fakedScheduler{}
+	scheduler := &scheduler.Scheduler{}
 	c := &fakedPolicyController{}
 	ctl := &controller{
 		scheduler: scheduler,
@@ -275,10 +266,13 @@ func TestUpdate(t *testing.T) {
 	current = origin
 	err = ctl.Update(current)
 	require.Nil(t, err)
-	assert.False(t, scheduler.scheduled)
-	assert.False(t, scheduler.unscheduled)
 
 	// the trigger changed
+	scheduler.On("Schedule", mock.Anything, mock.Anything,
+		mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil)
+	scheduler.On("UnScheduleByVendor", mock.Anything, mock.Anything,
+		mock.Anything).Return(nil)
+
 	origin = &model.Policy{
 		ID:      1,
 		Enabled: true,
@@ -301,12 +295,11 @@ func TestUpdate(t *testing.T) {
 	}
 	err = ctl.Update(current)
 	require.Nil(t, err)
-	assert.True(t, scheduler.unscheduled)
+	scheduler.AssertExpectations(t)
-	assert.True(t, scheduler.scheduled)
 }
 
 func TestRemove(t *testing.T) {
-	scheduler := &fakedScheduler{}
+	scheduler := &scheduler.Scheduler{}
 	c := &fakedPolicyController{}
 	ctl := &controller{
 		scheduler: scheduler,
@@ -328,9 +321,10 @@ func TestRemove(t *testing.T) {
 	c.policy = policy
 	err = ctl.Remove(1)
 	require.Nil(t, err)
-	assert.False(t, scheduler.unscheduled)
 
 	// the trigger type is scheduled
+	scheduler.On("UnScheduleByVendor", mock.Anything, mock.Anything,
+		mock.Anything).Return(nil)
 	policy = &model.Policy{
 		ID:      1,
 		Enabled: true,
@@ -344,5 +338,5 @@ func TestRemove(t *testing.T) {
 	c.policy = policy
 	err = ctl.Remove(1)
 	require.Nil(t, err)
-	assert.True(t, scheduler.unscheduled)
+	scheduler.AssertExpectations(t)
 }

src/replication/policy/scheduler/scheduler.go

@@ -1,121 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package scheduler
-
-import (
-	"fmt"
-	"net/http"
-	"strings"
-	"time"
-
-	commonHttp "github.com/goharbor/harbor/src/common/http"
-	"github.com/goharbor/harbor/src/common/job"
-	jobModels "github.com/goharbor/harbor/src/common/job/models"
-	jsJob "github.com/goharbor/harbor/src/jobservice/job"
-	"github.com/goharbor/harbor/src/lib/log"
-	"github.com/goharbor/harbor/src/replication/config"
-	"github.com/goharbor/harbor/src/replication/dao"
-	"github.com/goharbor/harbor/src/replication/dao/models"
-)
-
-// Scheduler can be used to schedule or unschedule a scheduled policy
-// Currently, the default scheduler implements its capabilities by delegating
-// the scheduled job of jobservice
-type Scheduler interface {
-	Schedule(policyID int64, cron string) error
-	Unschedule(policyID int64) error
-}
-
-// NewScheduler returns an instance of scheduler
-func NewScheduler(js job.Client) Scheduler {
-	return &scheduler{
-		jobservice: js,
-	}
-}
-
-type scheduler struct {
-	jobservice job.Client
-}
-
-func (s *scheduler) Schedule(policyID int64, cron string) error {
-	now := time.Now()
-	id, err := dao.ScheduleJob.Add(&models.ScheduleJob{
-		PolicyID:     policyID,
-		Status:       job.JobServiceStatusPending,
-		CreationTime: now,
-		UpdateTime:   now,
-	})
-	if err != nil {
-		return err
-	}
-	log.Debugf("the schedule job record %d added", id)
-
-	statusHookURL := fmt.Sprintf("%s/service/notifications/jobs/replication/%d", config.Config.CoreURL, id)
-	jobID, err := s.jobservice.SubmitJob(&jobModels.JobData{
-		Name: jsJob.ReplicationScheduler,
-		Parameters: map[string]interface{}{
-			"url":       config.Config.CoreURL,
-			"policy_id": policyID,
-		},
-		Metadata: &jobModels.JobMetadata{
-			JobKind: job.JobKindPeriodic,
-			Cron:    cron,
-		},
-		StatusHook: statusHookURL,
-	})
-	if err != nil {
-		// clean up the record in database
-		if e := dao.ScheduleJob.Delete(id); e != nil {
-			log.Errorf("failed to delete the schedule job %d: %v", id, e)
-		} else {
-			log.Debugf("the schedule job record %d deleted", id)
-		}
-		return err
-	}
-	log.Debugf("the schedule job for policy %d submitted to the jobservice", policyID)
-
-	err = dao.ScheduleJob.Update(&models.ScheduleJob{
-		ID:    id,
-		JobID: jobID,
-	}, "JobID")
-	log.Debugf("the policy %d scheduled", policyID)
-	return err
-}
-
-func (s *scheduler) Unschedule(policyID int64) error {
-	sjs, err := dao.ScheduleJob.List(&models.ScheduleJobQuery{
-		PolicyID: policyID,
-	})
-	if err != nil {
-		return err
-	}
-	for _, sj := range sjs {
-		if err = s.jobservice.PostAction(sj.JobID, job.JobActionStop); err != nil {
-			// if the job specified by jobID is not found in jobservice, just delete
-			// the record from database
-			if e, ok := err.(*commonHttp.Error); !ok || (e.Code != http.StatusNotFound &&
-				!strings.Contains(e.Message, "no valid periodic job policy found")) {
-				return err
-			}
-			log.Debugf("the stop action for schedule job %s submitted to the jobservice", sj.JobID)
-		}
-		if err = dao.ScheduleJob.Delete(sj.ID); err != nil {
-			return err
-		}
-		log.Debugf("the schedule job record %d deleted", sj.ID)
-	}
-	log.Debugf("the policy %d unscheduled", policyID)
-	return nil
-}

src/replication/policy/scheduler/scheduler_test.go

@@ -1,172 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package scheduler
-
-import (
-	"fmt"
-	"testing"
-
-	"github.com/goharbor/harbor/src/common/job/models"
-	"github.com/goharbor/harbor/src/jobservice/job"
-	"github.com/goharbor/harbor/src/replication/config"
-	"github.com/goharbor/harbor/src/replication/dao"
-	rep_models "github.com/goharbor/harbor/src/replication/dao/models"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// TODO share the faked implementation in a separated common package?
-// TODO can we use a mock framework?
-
-var (
-	uuid           = "uuid"
-	policyID int64 = 100
-)
-
-type fakedJobserviceClient struct {
-	jobData *models.JobData
-	stopped bool
-}
-
-func (f *fakedJobserviceClient) SubmitJob(jobData *models.JobData) (string, error) {
-	f.jobData = jobData
-	return uuid, nil
-}
-func (f *fakedJobserviceClient) GetJobLog(uuid string) ([]byte, error) {
-	f.stopped = true
-	return nil, nil
-}
-func (f *fakedJobserviceClient) PostAction(uuid, action string) error {
-	f.stopped = true
-	return nil
-}
-func (f *fakedJobserviceClient) GetExecutions(uuid string) ([]job.Stats, error) {
-	f.stopped = true
-	return nil, nil
-}
-
-type fakedScheduleJobDAO struct {
-	idCounter int64
-	sjs       map[int64]*rep_models.ScheduleJob
-}
-
-func (f *fakedScheduleJobDAO) Add(sj *rep_models.ScheduleJob) (int64, error) {
-	if f.sjs == nil {
-		f.sjs = make(map[int64]*rep_models.ScheduleJob)
-	}
-	id := f.idCounter + 1
-	sj.ID = id
-	f.sjs[id] = sj
-	return id, nil
-}
-func (f *fakedScheduleJobDAO) Get(id int64) (*rep_models.ScheduleJob, error) {
-	if f.sjs == nil {
-		return nil, nil
-	}
-	return f.sjs[id], nil
-}
-func (f *fakedScheduleJobDAO) Update(sj *rep_models.ScheduleJob, props ...string) error {
-	err := fmt.Errorf("schedule job %d not found", sj.ID)
-	if f.sjs == nil {
-		return err
-	}
-	j, exist := f.sjs[sj.ID]
-	if !exist {
-		return err
-	}
-	if len(props) == 0 {
-		f.sjs[sj.ID] = sj
-		return nil
-	}
-
-	for _, prop := range props {
-		switch prop {
-		case "PolicyID":
-			j.PolicyID = sj.PolicyID
-		case "JobID":
-			j.JobID = sj.JobID
-		case "Status":
-			j.Status = sj.Status
-		case "UpdateTime":
-			j.UpdateTime = sj.UpdateTime
-		}
-	}
-	return nil
-}
-func (f *fakedScheduleJobDAO) Delete(id int64) error {
-	if f.sjs == nil {
-		return nil
-	}
-	delete(f.sjs, id)
-	return nil
-}
-func (f *fakedScheduleJobDAO) List(query ...*rep_models.ScheduleJobQuery) ([]*rep_models.ScheduleJob, error) {
-	var policyID int64
-	if len(query) > 0 {
-		policyID = query[0].PolicyID
-	}
-	sjs := []*rep_models.ScheduleJob{}
-	for _, sj := range f.sjs {
-		if policyID == 0 {
-			sjs = append(sjs, sj)
-			continue
-		}
-		if sj.PolicyID == policyID {
-			sjs = append(sjs, sj)
-		}
-	}
-	return sjs, nil
-}
-
-func TestSchedule(t *testing.T) {
-	config.Config = &config.Configuration{}
-	dao.ScheduleJob = &fakedScheduleJobDAO{}
-	js := &fakedJobserviceClient{}
-	scheduler := NewScheduler(js)
-	err := scheduler.Schedule(policyID, "1 * * * *")
-	require.Nil(t, err)
-
-	sjs, err := dao.ScheduleJob.List(&rep_models.ScheduleJobQuery{
-		PolicyID: policyID,
-	})
-	require.Nil(t, err)
-	require.Equal(t, 1, len(sjs))
-	assert.Equal(t, uuid, sjs[0].JobID)
-
-	policyID, ok := js.jobData.Parameters["policy_id"].(int64)
-	require.True(t, ok)
-	assert.Equal(t, policyID, policyID)
-}
-
-func TestUnschedule(t *testing.T) {
-	config.Config = &config.Configuration{}
-	dao.ScheduleJob = &fakedScheduleJobDAO{}
-	_, err := dao.ScheduleJob.Add(&rep_models.ScheduleJob{
-		PolicyID: policyID,
-	})
-	require.Nil(t, err)
-	js := &fakedJobserviceClient{}
-	scheduler := NewScheduler(js)
-	err = scheduler.Unschedule(policyID)
-	require.Nil(t, err)
-
-	sjs, err := dao.ScheduleJob.List(&rep_models.ScheduleJobQuery{
-		PolicyID: policyID,
-	})
-	require.Nil(t, err)
-	require.Equal(t, 0, len(sjs))
-
-	assert.True(t, js.stopped)
-}

src/replication/replication.go

@@ -15,14 +15,18 @@
 package replication
 
 import (
+	"context"
+	"fmt"
+	"strconv"
 	"time"
 
-	"github.com/goharbor/harbor/src/common/job"
+	"github.com/goharbor/harbor/src/controller/replication"
 	cfg "github.com/goharbor/harbor/src/core/config"
 	"github.com/goharbor/harbor/src/lib/log"
+	"github.com/goharbor/harbor/src/pkg/scheduler"
+	"github.com/goharbor/harbor/src/pkg/task"
 	"github.com/goharbor/harbor/src/replication/config"
 	"github.com/goharbor/harbor/src/replication/event"
-	"github.com/goharbor/harbor/src/replication/operation"
 	"github.com/goharbor/harbor/src/replication/policy"
 	"github.com/goharbor/harbor/src/replication/policy/controller"
 	"github.com/goharbor/harbor/src/replication/registry"
@@ -60,12 +64,37 @@ var (
 	PolicyCtl policy.Controller
 	// RegistryMgr is a global registry manager
 	RegistryMgr registry.Manager
-	// OperationCtl is a global operation controller
-	OperationCtl operation.Controller
 	// EventHandler handles images/chart pull/push events
 	EventHandler event.Handler
 )
 
+func init() {
+	callbackFunc := func(ctx context.Context, param string) error {
+		policyID, err := strconv.ParseInt(param, 10, 64)
+		if err != nil {
+			return err
+		}
+
+		policy, err := PolicyCtl.Get(policyID)
+		if err != nil {
+			return err
+		}
+		if policy == nil {
+			return fmt.Errorf("policy %d not found", policyID)
+		}
+		if err = event.PopulateRegistries(RegistryMgr, policy); err != nil {
+			return err
+		}
+
+		_, err = replication.Ctl.Start(ctx, policy, nil, task.ExecutionTriggerSchedule)
+		return err
+	}
+	err := scheduler.RegisterCallbackFunc(controller.CallbackFuncName, callbackFunc)
+	if err != nil {
+		log.Errorf("failed to register the callback function for replication: %v", err)
+	}
+}
+
 // Init the global variables and configurations
 func Init(closing, done chan struct{}) error {
 	// init config
@@ -76,21 +105,15 @@ func Init(closing, done chan struct{}) error {
 	config.Config = &config.Configuration{
 		CoreURL:          cfg.InternalCoreURL(),
 		TokenServiceURL:  cfg.InternalTokenServiceEndpoint(),
-		JobserviceURL:    cfg.InternalJobServiceURL(),
 		SecretKey:        secretKey,
-		CoreSecret:       cfg.CoreSecret(),
 		JobserviceSecret: cfg.JobserviceSecret(),
 	}
-	// TODO use a global http transport
-	js := job.NewDefaultClient(config.Config.JobserviceURL, config.Config.CoreSecret)
 	// init registry manager
 	RegistryMgr = registry.NewDefaultManager()
 	// init policy controller
-	PolicyCtl = controller.NewController(js)
+	PolicyCtl = controller.NewController()
-	// init operation controller
-	OperationCtl = operation.NewController(js)
 	// init event handler
-	EventHandler = event.NewHandler(PolicyCtl, RegistryMgr, OperationCtl)
+	EventHandler = event.NewHandler(PolicyCtl, RegistryMgr)
 	log.Debug("the replication initialization completed")
 
 	// Start health checker for registries

src/replication/replication_test.go

@@ -38,6 +38,5 @@ func TestInit(t *testing.T) {
 	require.Nil(t, err)
 	assert.NotNil(t, PolicyCtl)
 	assert.NotNil(t, RegistryMgr)
-	assert.NotNil(t, OperationCtl)
 	assert.NotNil(t, EventHandler)
 }

src/server/handler/job_status_hook.go

@@ -17,20 +17,18 @@ package handler
 import (
 	"encoding/json"
 	"net/http"
-	"strconv"
 
 	"github.com/goharbor/harbor/src/jobservice/job"
 	"github.com/goharbor/harbor/src/lib/errors"
 	libhttp "github.com/goharbor/harbor/src/lib/http"
 	"github.com/goharbor/harbor/src/lib/log"
 	"github.com/goharbor/harbor/src/pkg/task"
-	"github.com/goharbor/harbor/src/server/router"
 )
 
 // NewJobStatusHandler creates a handler to handle the job status changing
 func NewJobStatusHandler() http.Handler {
 	return &jobStatusHandler{
-		handler: task.NewHookHandler(),
+		handler: task.HkHandler,
 	}
 }
 
@@ -40,23 +38,16 @@ type jobStatusHandler struct {
 
 func (j *jobStatusHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	defer r.Body.Close()
-	taskIDParam := router.Param(r.Context(), ":id")
-	taskID, err := strconv.ParseInt(taskIDParam, 10, 64)
-	if err != nil {
-		libhttp.SendError(w, err)
-		return
-	}
 
 	sc := &job.StatusChange{}
-	if err = json.NewDecoder(r.Body).Decode(sc); err != nil {
+	if err := json.NewDecoder(r.Body).Decode(sc); err != nil {
 		libhttp.SendError(w, err)
 		return
 	}
-
+	if err := j.handler.Handle(r.Context(), sc); err != nil {
-	if err = j.handler.Handle(r.Context(), taskID, sc); err != nil {
 		// ignore the not found error to avoid the jobservice re-sending the hook
 		if errors.IsNotFoundErr(err) {
-			log.Warningf("task %d does not exist, ignore the not found error to avoid subsequent retrying webhooks from jobservice", taskID)
+			log.Warningf("got not found error: %v, ignore it to avoid subsequent retrying webhooks from jobservice", err)
 			return
 		}
 		libhttp.SendError(w, err)

src/server/route.go

@@ -15,6 +15,8 @@
 package server
 
 import (
+	"net/http"
+
 	"github.com/astaxie/beego"
 	"github.com/goharbor/harbor/src/common"
 	"github.com/goharbor/harbor/src/core/api"
@@ -22,11 +24,9 @@ import (
 	"github.com/goharbor/harbor/src/core/controllers"
 	"github.com/goharbor/harbor/src/core/service/notifications/admin"
 	"github.com/goharbor/harbor/src/core/service/notifications/jobs"
-	"github.com/goharbor/harbor/src/core/service/notifications/scheduler"
 	"github.com/goharbor/harbor/src/core/service/token"
 	"github.com/goharbor/harbor/src/server/handler"
 	"github.com/goharbor/harbor/src/server/router"
-	"net/http"
 )
 
 func registerRoutes() {
@@ -47,12 +47,11 @@ func registerRoutes() {
 	beego.Router("/api/internal/syncquota", &api.InternalAPI{}, "post:SyncQuota")
 
 	beego.Router("/service/notifications/jobs/adminjob/:id([0-9]+)", &admin.Handler{}, "post:HandleAdminJob")
-	beego.Router("/service/notifications/jobs/replication/:id([0-9]+)", &jobs.Handler{}, "post:HandleReplicationScheduleJob")
-	beego.Router("/service/notifications/jobs/replication/task/:id([0-9]+)", &jobs.Handler{}, "post:HandleReplicationTask")
 	beego.Router("/service/notifications/jobs/webhook/:id([0-9]+)", &jobs.Handler{}, "post:HandleNotificationJob")
 	beego.Router("/service/notifications/jobs/retention/task/:id([0-9]+)", &jobs.Handler{}, "post:HandleRetentionTask")
-	beego.Router("/service/notifications/schedules/:id([0-9]+)", &scheduler.Handler{}, "post:Handle")
 	beego.Router("/service/notifications/jobs/scan/:uuid", &jobs.Handler{}, "post:HandleScan")
+	router.NewRoute().Method(http.MethodPost).Path("/service/notifications/schedules/:id([0-9]+)").Handler(handler.NewJobStatusHandler())        // legacy job status hook endpoint for scheduler
+	router.NewRoute().Method(http.MethodPost).Path("/service/notifications/jobs/replication/:id([0-9]+)").Handler(handler.NewJobStatusHandler()) // legacy job status hook endpoint for replication scheduler
 	router.NewRoute().Method(http.MethodPost).Path("/service/notifications/tasks/:id").Handler(handler.NewJobStatusHandler())
 
 	beego.Router("/service/token", &token.Handler{})

src/server/v2.0/handler/handler.go

@@ -36,6 +36,7 @@ func New() http.Handler {
 		ProjectAPI:     newProjectAPI(),
 		PreheatAPI:     newPreheatAPI(),
 		IconAPI:        newIconAPI(),
+		ReplicationAPI: newReplicationAPI(),
 	})
 	if err != nil {
 		log.Fatal(err)

src/server/v2.0/handler/preheat.go

@@ -663,7 +663,7 @@ func convertTaskToPayload(model *task.Task) (*models.Task, error) {
 		ExecutionID:   model.ExecutionID,
 		ExtraAttrs:    model.ExtraAttrs,
 		ID:            model.ID,
-		RunCount:      int64(model.RunCount),
+		RunCount:      model.RunCount,
 		StartTime:     model.StartTime.String(),
 		Status:        model.Status,
 		StatusMessage: model.StatusMessage,

src/server/v2.0/handler/replication.go

@@ -0,0 +1,297 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package handler
+
+import (
+	"context"
+	"strconv"
+	"strings"
+
+	"github.com/go-openapi/runtime/middleware"
+	"github.com/go-openapi/strfmt"
+	"github.com/goharbor/harbor/src/controller/replication"
+	"github.com/goharbor/harbor/src/jobservice/job"
+	"github.com/goharbor/harbor/src/lib/errors"
+	"github.com/goharbor/harbor/src/lib/q"
+	"github.com/goharbor/harbor/src/pkg/task"
+	replica "github.com/goharbor/harbor/src/replication"
+	"github.com/goharbor/harbor/src/replication/event"
+	"github.com/goharbor/harbor/src/replication/policy"
+	"github.com/goharbor/harbor/src/replication/policy/manager"
+	"github.com/goharbor/harbor/src/server/v2.0/models"
+	operation "github.com/goharbor/harbor/src/server/v2.0/restapi/operations/replication"
+)
+
+func newReplicationAPI() *replicationAPI {
+	return &replicationAPI{
+		ctl:       replication.Ctl,
+		policyMgr: manager.NewDefaultManager(),
+	}
+}
+
+type replicationAPI struct {
+	BaseAPI
+	ctl       replication.Controller
+	policyMgr policy.Controller
+}
+
+func (r *replicationAPI) Prepare(ctx context.Context, operation string, params interface{}) middleware.Responder {
+	if err := r.RequireSysAdmin(ctx); err != nil {
+		r.SendError(ctx, err)
+	}
+	return nil
+}
+
+func (r *replicationAPI) StartReplication(ctx context.Context, params operation.StartReplicationParams) middleware.Responder {
+	// TODO move the following logic to the replication controller after refactoring the policy management part with the new programming model
+	policy, err := r.policyMgr.Get(params.Execution.PolicyID)
+	if err != nil {
+		return r.SendError(ctx, err)
+	}
+	if policy == nil {
+		return r.SendError(ctx, errors.New(nil).WithCode(errors.NotFoundCode).
+			WithMessage("the replication policy %d not found", params.Execution.PolicyID))
+	}
+	if err = event.PopulateRegistries(replica.RegistryMgr, policy); err != nil {
+		return r.SendError(ctx, err)
+	}
+
+	// the legacy replication scheduler job("src/jobservice/job/impl/replication/scheduler.go") calls the start replication API
+	// to trigger the scheduled replication, a query string "trigger" is added when sending the request
+	// here is the logic to cover this part
+	trigger := task.ExecutionTriggerManual
+	if params.HTTPRequest.URL.Query().Get("trigger") == "scheduled" {
+		trigger = task.ExecutionTriggerSchedule
+	}
+
+	executionID, err := r.ctl.Start(ctx, policy, nil, trigger)
+	if err != nil {
+		return r.SendError(ctx, err)
+	}
+	location := strings.TrimSuffix(params.HTTPRequest.URL.Path, "/") + "/" + strconv.FormatInt(executionID, 10)
+	return operation.NewStartReplicationCreated().WithLocation(location)
+}
+
+func (r *replicationAPI) StopReplication(ctx context.Context, params operation.StopReplicationParams) middleware.Responder {
+	if err := r.ctl.Stop(ctx, params.ID); err != nil {
+		return r.SendError(ctx, err)
+	}
+	return nil
+}
+
+func (r *replicationAPI) ListReplicationExecutions(ctx context.Context, params operation.ListReplicationExecutionsParams) middleware.Responder {
+	query, err := r.BuildQuery(ctx, nil, params.Page, params.PageSize)
+	if err != nil {
+		return r.SendError(ctx, err)
+	}
+	if params.PolicyID != nil {
+		query.Keywords["PolicyID"] = *params.PolicyID
+	}
+	if params.Status != nil {
+		status := *params.Status
+		// as we convert the status when responding requests to keep the backward compatibility,
+		// here we need to reverse-convert the status
+		switch status {
+		case "InProgress":
+			status = job.RunningStatus.String()
+		case "Succeed":
+			status = job.SuccessStatus.String()
+		case "Stopped":
+			status = job.StoppedStatus.String()
+		case "Failed":
+			status = job.ErrorStatus.String()
+		}
+		query.Keywords["Status"] = status
+	}
+	if params.Trigger != nil {
+		trigger := *params.Trigger
+		// as we convert the trigger when responding requests to keep the backward compatibility,
+		// here we need to reverse-convert the trigger
+		switch trigger {
+		case "manual":
+			trigger = task.ExecutionTriggerManual
+		case "scheduled":
+			trigger = task.ExecutionTriggerSchedule
+		case "event_based":
+			trigger = task.ExecutionTriggerEvent
+		}
+		query.Keywords["Trigger"] = trigger
+	}
+
+	total, err := r.ctl.ExecutionCount(ctx, query)
+	if err != nil {
+		return r.SendError(ctx, err)
+	}
+	executions, err := r.ctl.ListExecutions(ctx, query)
+	if err != nil {
+		return r.SendError(ctx, err)
+	}
+
+	var execs []*models.ReplicationExecution
+	for _, execution := range executions {
+		execs = append(execs, convertExecution(execution))
+	}
+
+	return operation.NewListReplicationExecutionsOK().
+		WithXTotalCount(total).
+		WithLink(r.Links(ctx, params.HTTPRequest.URL, total, query.PageNumber, query.PageSize).String()).
+		WithPayload(execs)
+}
+
+func (r *replicationAPI) GetReplicationExecution(ctx context.Context, params operation.GetReplicationExecutionParams) middleware.Responder {
+	execution, err := r.ctl.GetExecution(ctx, params.ID)
+	if err != nil {
+		return r.SendError(ctx, err)
+	}
+	return operation.NewGetReplicationExecutionOK().WithPayload(convertExecution(execution))
+}
+
+func (r *replicationAPI) ListReplicationTasks(ctx context.Context, params operation.ListReplicationTasksParams) middleware.Responder {
+	query, err := r.BuildQuery(ctx, nil, params.Page, params.PageSize)
+	if err != nil {
+		return r.SendError(ctx, err)
+	}
+	query.Keywords["ExecutionID"] = params.ID
+	if params.Status != nil {
+		var status interface{} = *params.Status
+		// as we convert the status when responding requests to keep the backward compatibility,
+		// here we need to reverse-convert the status
+		// the status "pending" and "stopped" is same with jobservice, no need to convert
+		switch status {
+		case "InProgress":
+			status = &q.OrList{
+				Values: []interface{}{
+					job.ScheduledStatus.String(),
+					job.RunningStatus.String(),
+				},
+			}
+		case "Succeed":
+			status = job.SuccessStatus.String()
+		case "Failed":
+			status = job.ErrorStatus.String()
+		}
+		query.Keywords["Status"] = status
+	}
+	if params.ResourceType != nil {
+		query.Keywords["ExtraAttrs.resource_type"] = *params.ResourceType
+	}
+
+	total, err := r.ctl.TaskCount(ctx, query)
+	if err != nil {
+		return r.SendError(ctx, err)
+	}
+
+	tasks, err := r.ctl.ListTasks(ctx, query)
+	if err != nil {
+		return r.SendError(ctx, err)
+	}
+
+	var tks []*models.ReplicationTask
+	for _, task := range tasks {
+		tks = append(tks, convertTask(task))
+	}
+
+	return operation.NewListReplicationTasksOK().
+		WithXTotalCount(total).
+		WithLink(r.Links(ctx, params.HTTPRequest.URL, total, query.PageNumber, query.PageSize).String()).
+		WithPayload(tks)
+}
+
+func (r *replicationAPI) GetReplicationLog(ctx context.Context, params operation.GetReplicationLogParams) middleware.Responder {
+	execution, err := r.ctl.GetExecution(ctx, params.ID)
+	if err != nil {
+		return r.SendError(ctx, err)
+	}
+	task, err := r.ctl.GetTask(ctx, params.TaskID)
+	if err != nil {
+		return r.SendError(ctx, err)
+	}
+	if execution.ID != task.ExecutionID {
+		return r.SendError(ctx, errors.New(nil).
+			WithCode(errors.NotFoundCode).
+			WithMessage("execution %d contains no task with ID %d", params.ID, params.TaskID))
+	}
+	log, err := r.ctl.GetTaskLog(ctx, params.TaskID)
+	if err != nil {
+		return r.SendError(ctx, err)
+	}
+	return operation.NewGetReplicationLogOK().WithContentType("text/plain").WithPayload(string(log))
+}
+
+func convertExecution(execution *replication.Execution) *models.ReplicationExecution {
+	exec := &models.ReplicationExecution{
+		ID:         execution.ID,
+		PolicyID:   execution.PolicyID,
+		StatusText: execution.StatusMessage,
+		StartTime:  strfmt.DateTime(execution.StartTime),
+		EndTime:    strfmt.DateTime(execution.EndTime),
+	}
+	// keep backward compatibility
+	if execution.Metrics != nil {
+		exec.Total = execution.Metrics.TaskCount
+		exec.Succeed = execution.Metrics.SuccessTaskCount
+		exec.Failed = execution.Metrics.ErrorTaskCount
+		exec.InProgress = execution.Metrics.PendingTaskCount +
+			execution.Metrics.ScheduledTaskCount + execution.Metrics.RunningTaskCount
+		exec.Stopped = execution.Metrics.StoppedTaskCount
+	}
+	switch execution.Trigger {
+	case task.ExecutionTriggerManual:
+		exec.Trigger = "manual"
+	case task.ExecutionTriggerSchedule:
+		exec.Trigger = "scheduled"
+	case task.ExecutionTriggerEvent:
+		exec.Trigger = "event_based"
+	}
+	switch execution.Status {
+	case job.RunningStatus.String():
+		exec.Status = "InProgress"
+	case job.SuccessStatus.String():
+		exec.Status = "Succeed"
+	case job.StoppedStatus.String():
+		exec.Status = "Stopped"
+	case job.ErrorStatus.String():
+		exec.Status = "Failed"
+	}
+
+	return exec
+}
+
+func convertTask(task *replication.Task) *models.ReplicationTask {
+	tk := &models.ReplicationTask{
+		ID:           task.ID,
+		ExecutionID:  task.ExecutionID,
+		JobID:        task.JobID,
+		Operation:    task.Operation,
+		ResourceType: task.ResourceType,
+		SrcResource:  task.SourceResource,
+		DstResource:  task.DestinationResource,
+		StartTime:    strfmt.DateTime(task.StartTime),
+		EndTime:      strfmt.DateTime(task.EndTime),
+	}
+	// keep backward compatibility
+	switch task.Status {
+	case job.ScheduledStatus.String(), job.RunningStatus.String():
+		tk.Status = "InProgress"
+	case job.SuccessStatus.String():
+		tk.Status = "Succeed"
+	case job.ErrorStatus.String():
+		tk.Status = "Failed"
+	// the status "pending" and "stopped" is same with jobservice, no need to convert
+	default:
+		tk.Status = task.Status
+	}
+	return tk
+}

src/server/v2.0/route/legacy.go

@@ -58,10 +58,6 @@ func registerLegacyRoutes() {
 
 	beego.Router("/api/"+version+"/replication/adapters", &api.ReplicationAdapterAPI{}, "get:List")
 	beego.Router("/api/"+version+"/replication/adapterinfos", &api.ReplicationAdapterAPI{}, "get:ListAdapterInfos")
-	beego.Router("/api/"+version+"/replication/executions", &api.ReplicationOperationAPI{}, "get:ListExecutions;post:CreateExecution")
-	beego.Router("/api/"+version+"/replication/executions/:id([0-9]+)", &api.ReplicationOperationAPI{}, "get:GetExecution;put:StopExecution")
-	beego.Router("/api/"+version+"/replication/executions/:id([0-9]+)/tasks", &api.ReplicationOperationAPI{}, "get:ListTasks")
-	beego.Router("/api/"+version+"/replication/executions/:id([0-9]+)/tasks/:tid([0-9]+)/log", &api.ReplicationOperationAPI{}, "get:GetTaskLog")
 	beego.Router("/api/"+version+"/replication/policies", &api.ReplicationPolicyAPI{}, "get:List;post:Create")
 	beego.Router("/api/"+version+"/replication/policies/:id([0-9]+)", &api.ReplicationPolicyAPI{}, "get:Get;put:Update;delete:Delete")
 

src/testing/controller/controller.go

@@ -21,3 +21,4 @@ package controller
 //go:generate mockery --case snake --dir ../../controller/scan --name Controller --output ./scan --outpkg scan
 //go:generate mockery --case snake --dir ../../controller/scan --name Checker --output ./scan --outpkg scan
 //go:generate mockery --case snake --dir ../../controller/scanner --name Controller --output ./scanner --outpkg scanner
+//go:generate mockery --case snake --dir ../../controller/replication --name Controller --output ./replication --outpkg replication

src/testing/controller/replication/controller.go

@@ -0,0 +1,211 @@
+// Code generated by mockery v2.1.0. DO NOT EDIT.
+
+package replication
+
+import (
+	context "context"
+
+	model "github.com/goharbor/harbor/src/replication/model"
+	mock "github.com/stretchr/testify/mock"
+
+	q "github.com/goharbor/harbor/src/lib/q"
+
+	replication "github.com/goharbor/harbor/src/controller/replication"
+)
+
+// Controller is an autogenerated mock type for the Controller type
+type Controller struct {
+	mock.Mock
+}
+
+// ExecutionCount provides a mock function with given fields: ctx, query
+func (_m *Controller) ExecutionCount(ctx context.Context, query *q.Query) (int64, error) {
+	ret := _m.Called(ctx, query)
+
+	var r0 int64
+	if rf, ok := ret.Get(0).(func(context.Context, *q.Query) int64); ok {
+		r0 = rf(ctx, query)
+	} else {
+		r0 = ret.Get(0).(int64)
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(context.Context, *q.Query) error); ok {
+		r1 = rf(ctx, query)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// GetExecution provides a mock function with given fields: ctx, executionID
+func (_m *Controller) GetExecution(ctx context.Context, executionID int64) (*replication.Execution, error) {
+	ret := _m.Called(ctx, executionID)
+
+	var r0 *replication.Execution
+	if rf, ok := ret.Get(0).(func(context.Context, int64) *replication.Execution); ok {
+		r0 = rf(ctx, executionID)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*replication.Execution)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(context.Context, int64) error); ok {
+		r1 = rf(ctx, executionID)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// GetTask provides a mock function with given fields: ctx, taskID
+func (_m *Controller) GetTask(ctx context.Context, taskID int64) (*replication.Task, error) {
+	ret := _m.Called(ctx, taskID)
+
+	var r0 *replication.Task
+	if rf, ok := ret.Get(0).(func(context.Context, int64) *replication.Task); ok {
+		r0 = rf(ctx, taskID)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*replication.Task)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(context.Context, int64) error); ok {
+		r1 = rf(ctx, taskID)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// GetTaskLog provides a mock function with given fields: ctx, taskID
+func (_m *Controller) GetTaskLog(ctx context.Context, taskID int64) ([]byte, error) {
+	ret := _m.Called(ctx, taskID)
+
+	var r0 []byte
+	if rf, ok := ret.Get(0).(func(context.Context, int64) []byte); ok {
+		r0 = rf(ctx, taskID)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).([]byte)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(context.Context, int64) error); ok {
+		r1 = rf(ctx, taskID)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// ListExecutions provides a mock function with given fields: ctx, query
+func (_m *Controller) ListExecutions(ctx context.Context, query *q.Query) ([]*replication.Execution, error) {
+	ret := _m.Called(ctx, query)
+
+	var r0 []*replication.Execution
+	if rf, ok := ret.Get(0).(func(context.Context, *q.Query) []*replication.Execution); ok {
+		r0 = rf(ctx, query)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).([]*replication.Execution)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(context.Context, *q.Query) error); ok {
+		r1 = rf(ctx, query)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// ListTasks provides a mock function with given fields: ctx, query
+func (_m *Controller) ListTasks(ctx context.Context, query *q.Query) ([]*replication.Task, error) {
+	ret := _m.Called(ctx, query)
+
+	var r0 []*replication.Task
+	if rf, ok := ret.Get(0).(func(context.Context, *q.Query) []*replication.Task); ok {
+		r0 = rf(ctx, query)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).([]*replication.Task)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(context.Context, *q.Query) error); ok {
+		r1 = rf(ctx, query)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// Start provides a mock function with given fields: ctx, policy, resource, trigger
+func (_m *Controller) Start(ctx context.Context, policy *model.Policy, resource *model.Resource, trigger string) (int64, error) {
+	ret := _m.Called(ctx, policy, resource, trigger)
+
+	var r0 int64
+	if rf, ok := ret.Get(0).(func(context.Context, *model.Policy, *model.Resource, string) int64); ok {
+		r0 = rf(ctx, policy, resource, trigger)
+	} else {
+		r0 = ret.Get(0).(int64)
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(context.Context, *model.Policy, *model.Resource, string) error); ok {
+		r1 = rf(ctx, policy, resource, trigger)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// Stop provides a mock function with given fields: ctx, executionID
+func (_m *Controller) Stop(ctx context.Context, executionID int64) error {
+	ret := _m.Called(ctx, executionID)
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func(context.Context, int64) error); ok {
+		r0 = rf(ctx, executionID)
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+// TaskCount provides a mock function with given fields: ctx, query
+func (_m *Controller) TaskCount(ctx context.Context, query *q.Query) (int64, error) {
+	ret := _m.Called(ctx, query)
+
+	var r0 int64
+	if rf, ok := ret.Get(0).(func(context.Context, *q.Query) int64); ok {
+		r0 = rf(ctx, query)
+	} else {
+		r0 = ret.Get(0).(int64)
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(context.Context, *q.Query) error); ok {
+		r1 = rf(ctx, query)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}

src/testing/lib/lib.go

@@ -12,17 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package scheduler
+package lib
 
-import (
+//go:generate mockery --case snake --dir ../../lib/orm --name Creator --output ./orm --outpkg orm
-	"github.com/goharbor/harbor/src/replication/dao"
-	"github.com/goharbor/harbor/src/replication/dao/models"
-)
-
-// UpdateStatus updates the schedule job status
-func UpdateStatus(id int64, status string) error {
-	return dao.ScheduleJob.Update(&models.ScheduleJob{
-		ID:     id,
-		Status: status,
-	}, "Status")
-}

src/testing/lib/orm/creator.go

@@ -0,0 +1,29 @@
+// Code generated by mockery v2.1.0. DO NOT EDIT.
+
+package orm
+
+import (
+	orm "github.com/astaxie/beego/orm"
+	mock "github.com/stretchr/testify/mock"
+)
+
+// Creator is an autogenerated mock type for the Creator type
+type Creator struct {
+	mock.Mock
+}
+
+// Create provides a mock function with given fields:
+func (_m *Creator) Create() orm.Ormer {
+	ret := _m.Called()
+
+	var r0 orm.Ormer
+	if rf, ok := ret.Get(0).(func() orm.Ormer); ok {
+		r0 = rf()
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(orm.Ormer)
+		}
+	}
+
+	return r0
+}

tests/apitests/python/library/base.py

@@ -28,7 +28,7 @@ def get_endpoint():
 
 def _create_client(server, credential, debug, api_type="products"):
     cfg = None
-    if api_type in ('projectv2', 'artifact', 'repository', 'scan', 'preheat'):
+    if api_type in ('projectv2', 'artifact', 'repository', 'scan', 'preheat', 'replication'):
         cfg = v2_swagger_client.Configuration()
     else:
         cfg = swagger_client.Configuration()
@@ -59,6 +59,7 @@ def _create_client(server, credential, debug, api_type="products"):
         "repository": v2_swagger_client.RepositoryApi(v2_swagger_client.ApiClient(cfg)),
         "scan": v2_swagger_client.ScanApi(v2_swagger_client.ApiClient(cfg)),
         "scanner": swagger_client.ScannersApi(swagger_client.ApiClient(cfg)),
+        "replication": v2_swagger_client.ReplicationApi(v2_swagger_client.ApiClient(cfg)),
     }.get(api_type,'Error: Wrong API type')
 
 def _assert_status_code(expect_code, return_code):

tests/apitests/python/library/replication.py

@@ -45,41 +45,7 @@ class Replication(base.Base):
             #else:
             #    raise Exception(r"Check replication rule trigger failed, expect <{}> actual <{}>.".format(expect_trigger, get_trigger))
 
-
-    def start_replication(self, rule_id, **kwargs):
-        client = self._get_client(**kwargs)
-        return client.replications_post(swagger_client.Replication(int(rule_id)))
-
-    def list_replication_jobs(self, rule_id, **kwargs):
-        client = self._get_client(**kwargs)
-        return client.jobs_replication_get(int(rule_id))
-
-    def wait_until_jobs_finish(self, rule_id, retry=10, interval=5, **kwargs):
-        Succeed = False
-        for i in range(retry):
-            Succeed = False
-            jobs = self.get_replication_executions(rule_id, **kwargs)
-            for job in jobs:
-                if job.status == "Succeed":
-                    return
-            if not Succeed:
-                time.sleep(interval)
-        if not Succeed:
-            raise Exception("The jobs not Succeed")
-
     def delete_replication_rule(self, rule_id, expect_status_code = 200, **kwargs):
         client = self._get_client(**kwargs)
         _, status_code, _ = client.replication_policies_id_delete_with_http_info(rule_id)
         base._assert_status_code(expect_status_code, status_code)
-
-    def trigger_replication_executions(self, rule_id, expect_status_code = 201, **kwargs):
-        client = self._get_client(**kwargs)
-        _, status_code, _ = client.replication_executions_post_with_http_info({"policy_id":rule_id})
-        base._assert_status_code(expect_status_code, status_code)
-
-
-    def get_replication_executions(self, rule_id, expect_status_code = 200, **kwargs):
-        client = self._get_client(**kwargs)
-        data, status_code, _ = client.replication_executions_get_with_http_info(policy_id=rule_id)
-        base._assert_status_code(expect_status_code, status_code)
-        return data

tests/apitests/python/library/replication_v2.py

@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+
+import time
+import base
+import v2_swagger_client
+from v2_swagger_client.rest import ApiException
+
+class ReplicationV2(base.Base, object):
+    def __init__(self):
+        super(ReplicationV2,self).__init__(api_type = "replication")
+
+    def wait_until_jobs_finish(self, rule_id, retry=10, interval=5, **kwargs):
+        Succeed = False
+        for i in range(retry):
+            Succeed = False
+            jobs = self.get_replication_executions(rule_id, **kwargs)
+            for job in jobs:
+                if job.status == "Succeed":
+                    return
+            if not Succeed:
+                time.sleep(interval)
+        if not Succeed:
+            raise Exception("The jobs not Succeed")
+
+    def trigger_replication_executions(self, rule_id, expect_status_code = 201, **kwargs):
+        client = self._get_client(**kwargs)
+        _, status_code, _ = client.start_replication_with_http_info({"policy_id":rule_id})
+        base._assert_status_code(expect_status_code, status_code)
+
+    def get_replication_executions(self, rule_id, expect_status_code = 200, **kwargs):
+        client = self._get_client(**kwargs)
+        data, status_code, _ = client.list_replication_executions_with_http_info(policy_id=rule_id)
+        base._assert_status_code(expect_status_code, status_code)
+        return data
+

tests/apitests/python/test_replication_from_dockerhub.py

@@ -9,6 +9,7 @@ from library.replication import Replication
 from library.registry import Registry
 from library.artifact import Artifact
 from library.repository import Repository
+from library.replication_v2 import ReplicationV2
 import swagger_client
 from testutils import DOCKER_USER, DOCKER_PWD
 
@@ -18,6 +19,7 @@ class TestProjects(unittest.TestCase):
         self.project = Project()
         self.user = User()
         self.replication = Replication()
+        self.replication_v2 = ReplicationV2()
         self.registry = Registry()
         self.artifact = Artifact()
         self.repo = Repository()
@@ -88,10 +90,10 @@ class TestProjects(unittest.TestCase):
         self.replication.check_replication_rule_should_exist(TestProjects.rule_id, rule_name, **ADMIN_CLIENT)
 
         #6. Trigger the rule;
-        self.replication.trigger_replication_executions(TestProjects.rule_id, **ADMIN_CLIENT)
+        self.replication_v2.trigger_replication_executions(TestProjects.rule_id, **ADMIN_CLIENT)
 
         #7. Wait for completion of this replication job;
-        self.replication.wait_until_jobs_finish(TestProjects.rule_id,interval=30, **ADMIN_CLIENT)
+        self.replication_v2.wait_until_jobs_finish(TestProjects.rule_id,interval=30, **ADMIN_CLIENT)
 
         #8. Check image is replicated into target project successfully.
         artifact = self.artifact.get_reference_info(TestProjects.project_name, self.image, self.tag, **ADMIN_CLIENT)